Coverage for python/lsst/obs/lsst/_ingestPhotodiode.py: 17%

81 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-02 11:51 -0700

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21__all__ = ('PhotodiodeIngestConfig', 'PhotodiodeIngestTask') 

22 

23import warnings 

24 

25from lsst.daf.butler import ( 

26 CollectionType, 

27 DataCoordinate, 

28 DatasetIdGenEnum, 

29 DatasetRef, 

30 DatasetType, 

31 FileDataset, 

32 Progress, 

33 UnresolvedRefWarning, 

34) 

35from lsst.ip.isr import PhotodiodeCalib 

36from lsst.obs.base import makeTransferChoiceField 

37from lsst.obs.base.formatters.fitsGeneric import FitsGenericFormatter 

38from lsst.pex.config import Config 

39from lsst.pipe.base import Task 

40from lsst.resources import ResourcePath 

41 

42 

43class PhotodiodeIngestConfig(Config): 

44 """Configuration class for PhotodiodeIngestTask.""" 

45 

46 transfer = makeTransferChoiceField(default="copy") 

47 

48 def validate(self): 

49 super().validate() 

50 if self.transfer != "copy": 

51 raise ValueError(f"Transfer Must be 'copy' for photodiode data. {self.transfer}") 

52 

53 

54class PhotodiodeIngestTask(Task): 

55 """Task to ingest photodiode data into a butler repository. 

56 

57 Parameters 

58 ---------- 

59 config : `PhotodiodeIngestConfig` 

60 Configuration for the task. 

61 instrument : `~lsst.obs.base.Instrument` 

62 The instrument these photodiode datasets are from. 

63 butler : `~lsst.daf.butler.Butler` 

64 Writable butler instance, with ``butler.run`` set to the 

65 appropriate `~lsst.daf.butler.CollectionType.RUN` collection 

66 for these datasets. 

67 **kwargs 

68 Additional keyword arguments. 

69 """ 

70 

71 ConfigClass = PhotodiodeIngestConfig 

72 _DefaultName = "photodiodeIngest" 

73 

74 def getDatasetType(self): 

75 """Return the DatasetType of the photodiode datasets.""" 

76 return DatasetType( 

77 "photodiode", 

78 ("instrument", "exposure"), 

79 "IsrCalib", 

80 universe=self.butler.registry.dimensions, 

81 ) 

82 

83 def __init__(self, butler, instrument, config=None, **kwargs): 

84 config.validate() 

85 super().__init__(config, **kwargs) 

86 self.butler = butler 

87 self.universe = self.butler.registry.dimensions 

88 self.datasetType = self.getDatasetType() 

89 self.progress = Progress(self.log.name) 

90 self.instrument = instrument 

91 self.camera = self.instrument.getCamera() 

92 

93 def run(self, locations, run=None, file_filter=r".*Photodiode_Readings.*txt", 

94 track_file_attrs=None): 

95 """Ingest photodiode data into a Butler data repository. 

96 

97 Parameters 

98 ---------- 

99 files : iterable over `lsst.resources.ResourcePath` 

100 URIs to the files to be ingested. 

101 run : `str`, optional 

102 Name of the RUN-type collection to write to, 

103 overriding the default derived from the instrument 

104 name. 

105 skip_existing_exposures : `bool`, optional 

106 If `True`, skip photodiodes that have already been 

107 ingested (i.e. raws for which we already have a 

108 dataset with the same data ID in the target 

109 collection). 

110 track_file_attrs : `bool`, optional 

111 Control whether file attributes such as the size or 

112 checksum should be tracked by the datastore. Whether 

113 this parameter is honored depends on the specific 

114 datastore implementation. 

115 

116 Returns 

117 ------- 

118 refs : `list` [`lsst.daf.butler.DatasetRef`] 

119 Dataset references for ingested raws. 

120 

121 Raises 

122 ------ 

123 RuntimeError 

124 Raised if the number of exposures found for a photodiode 

125 file is not one 

126 """ 

127 files = ResourcePath.findFileResources(locations, file_filter) 

128 

129 registry = self.butler.registry 

130 registry.registerDatasetType(self.datasetType) 

131 

132 # Find and register run that we will ingest to. 

133 if run is None: 

134 run = self.instrument.makeCollectionName("calib", "photodiode") 

135 registry.registerCollection(run, type=CollectionType.RUN) 

136 

137 # Use datasetIds that match the raw exposure data. 

138 if self.butler.registry.supportsIdGenerationMode(DatasetIdGenEnum.DATAID_TYPE_RUN): 

139 mode = DatasetIdGenEnum.DATAID_TYPE_RUN 

140 else: 

141 mode = DatasetIdGenEnum.UNIQUE 

142 

143 refs = [] 

144 numExisting = 0 

145 numFailed = 0 

146 for inputFile in files: 

147 # Convert the file into the right class. 

148 with inputFile.as_local() as localFile: 

149 calib = PhotodiodeCalib.readTwoColumnPhotodiodeData(localFile.ospath) 

150 

151 dayObs = calib.getMetadata()['day_obs'] 

152 seqNum = calib.getMetadata()['seq_num'] 

153 

154 # Find the associated exposure information. 

155 whereClause = "exposure.day_obs=dayObs and exposure.seq_num=seqNum" 

156 instrumentName = self.instrument.getName() 

157 exposureRecords = [rec for rec in registry.queryDimensionRecords("exposure", 

158 instrument=instrumentName, 

159 where=whereClause, 

160 bind={"dayObs": dayObs, 

161 "seqNum": seqNum})] 

162 

163 nRecords = len(exposureRecords) 

164 if nRecords == 1: 

165 exposureId = exposureRecords[0].id 

166 calib.updateMetadata(camera=self.camera, exposure=exposureId) 

167 elif nRecords == 0: 

168 numFailed += 1 

169 self.log.warning("Skipping instrument %s and dayObs/seqNum %d %d: no exposures found.", 

170 instrumentName, dayObs, seqNum) 

171 continue 

172 else: 

173 numFailed += 1 

174 self.log.warning("Multiple exposure entries found for instrument %s and " 

175 "dayObs/seqNum %d %d.", instrumentName, dayObs, seqNum) 

176 continue 

177 

178 # Generate the dataId for this file. 

179 dataId = DataCoordinate.standardize( 

180 instrument=self.instrument.getName(), 

181 exposure=exposureId, 

182 universe=self.universe, 

183 ) 

184 

185 # If this already exists, we should skip it and continue. 

186 existing = { 

187 ref.dataId 

188 for ref in self.butler.registry.queryDatasets(self.datasetType, collections=[run], 

189 dataId=dataId) 

190 } 

191 if existing: 

192 self.log.debug("Skipping instrument %s and dayObs/seqNum %d %d: already exists in run %s.", 

193 instrumentName, dayObs, seqNum, run) 

194 numExisting += 1 

195 continue 

196 

197 # Ingest must work from a file, but we can't use the 

198 # original, as we've added new metadata and reformatted 

199 # it. Write it to a temp file that we can use to ingest. 

200 # If we can have the files written appropriately, this 

201 # will be a direct ingest of those files. 

202 with ResourcePath.temporary_uri(suffix=".fits") as tempFile: 

203 calib.writeFits(tempFile.ospath) 

204 

205 with warnings.catch_warnings(): 

206 warnings.simplefilter("ignore", category=UnresolvedRefWarning) 

207 ref = DatasetRef(self.datasetType, dataId) 

208 dataset = FileDataset(path=tempFile, refs=ref, formatter=FitsGenericFormatter) 

209 

210 # No try, as if this fails, we should stop. 

211 self.butler.ingest(dataset, transfer=self.config.transfer, run=run, 

212 idGenerationMode=mode, 

213 record_validation_info=track_file_attrs) 

214 self.log.info("Photodiode %s:%d (%d/%d) ingested successfully", instrumentName, exposureId, 

215 dayObs, seqNum) 

216 refs.append(dataset) 

217 

218 if numExisting != 0: 

219 self.log.warning("Skipped %d entries that already existed in run %s", numExisting, run) 

220 if numFailed != 0: 

221 raise RuntimeError(f"Failed to ingest {numFailed} entries due to missing exposure information.") 

222 return refs