Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_subaru. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Gen3 Butler registry declarations for Hyper Suprime-Cam. 

23""" 

24 

25__all__ = ("HyperSuprimeCam",) 

26 

27import os 

28import pickle 

29import logging 

30import datetime 

31from dateutil import parser 

32 

33from lsst.utils import getPackageDir 

34from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig 

35from lsst.daf.butler import DatasetType, DataCoordinate, FileDataset, DatasetRef 

36from lsst.obs.base import Instrument, addUnboundedCalibrationLabel 

37from lsst.pipe.tasks.read_curated_calibs import read_all 

38 

39from lsst.obs.hsc.hscPupil import HscPupilFactory 

40from lsst.obs.hsc.hscFilters import HSC_FILTER_DEFINITIONS 

41from lsst.obs.hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission, 

42 getFilterTransmission, getAtmosphereTransmission) 

43from lsst.obs.subaru.strayLight.formatter import SubaruStrayLightDataFormatter 

44 

45log = logging.getLogger(__name__) 

46 

47 

48class HyperSuprimeCam(Instrument): 

49 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam. 

50 """ 

51 

52 filterDefinitions = HSC_FILTER_DEFINITIONS 

53 

54 def __init__(self, **kwargs): 

55 super().__init__(**kwargs) 

56 packageDir = getPackageDir("obs_subaru") 

57 self.configPaths = [os.path.join(packageDir, "config"), 

58 os.path.join(packageDir, "config", "hsc")] 

59 

60 @classmethod 

61 def getName(cls): 

62 # Docstring inherited from Instrument.getName 

63 return "HSC" 

64 

65 def register(self, registry): 

66 # Docstring inherited from Instrument.register 

67 camera = self.getCamera() 

68 # The maximum values below make Gen3's ObservationDataIdPacker produce 

69 # outputs that match Gen2's ccdExposureId. 

70 obsMax = 21474800 

71 registry.insertDimensionData( 

72 "instrument", 

73 { 

74 "name": self.getName(), 

75 "detector_max": 200, 

76 "visit_max": obsMax, 

77 "exposure_max": obsMax 

78 } 

79 ) 

80 registry.insertDimensionData( 

81 "detector", 

82 *[ 

83 { 

84 "instrument": self.getName(), 

85 "id": detector.getId(), 

86 "full_name": detector.getName(), 

87 # TODO: make sure these definitions are consistent with those 

88 # extracted by astro_metadata_translator, and test that they 

89 # remain consistent somehow. 

90 "name_in_raft": detector.getName().split("_")[1], 

91 "raft": detector.getName().split("_")[0], 

92 "purpose": str(detector.getType()).split(".")[-1], 

93 } 

94 for detector in camera 

95 ] 

96 ) 

97 self._registerFilters(registry) 

98 

99 def getRawFormatter(self, dataId): 

100 # Docstring inherited from Instrument.getRawFormatter 

101 # Import the formatter here to prevent a circular dependency. 

102 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter 

103 if dataId["detector"] in (100, 101, 102, 103): 

104 return HyperSuprimeCamCornerRawFormatter 

105 else: 

106 return HyperSuprimeCamRawFormatter 

107 

108 def getCamera(self): 

109 """Retrieve the cameraGeom representation of HSC. 

110 

111 This is a temporary API that should go away once obs_ packages have 

112 a standardized approach to writing versioned cameras to a Gen3 repo. 

113 """ 

114 path = os.path.join(getPackageDir("obs_subaru"), "hsc", "camera") 

115 config = CameraConfig() 

116 config.load(os.path.join(path, "camera.py")) 

117 return makeCameraFromPath( 

118 cameraConfig=config, 

119 ampInfoPath=path, 

120 shortNameFunc=lambda name: name.replace(" ", "_"), 

121 pupilFactoryClass=HscPupilFactory 

122 ) 

123 

124 def getBrighterFatterKernel(self): 

125 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`. 

126 

127 This is a temporary API that should go away once obs_ packages have 

128 a standardized approach to writing versioned kernels to a Gen3 repo. 

129 """ 

130 path = os.path.join(getPackageDir("obs_subaru"), "hsc", "brighter_fatter_kernel.pkl") 

131 with open(path, "rb") as fd: 

132 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2 

133 return kernel 

134 

135 def writeCuratedCalibrations(self, butler): 

136 """Write human-curated calibration Datasets to the given Butler with 

137 the appropriate validity ranges. 

138 

139 This is a temporary API that should go away once obs_ packages have 

140 a standardized approach to this problem. 

141 """ 

142 

143 # Write cameraGeom.Camera, with an infinite validity range. 

144 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

145 universe=butler.registry.dimensions) 

146 butler.registry.registerDatasetType(datasetType) 

147 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

148 camera = self.getCamera() 

149 butler.put(camera, datasetType, unboundedDataId) 

150 

151 # Write brighter-fatter kernel, with an infinite validity range. 

152 datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray", 

153 universe=butler.registry.dimensions) 

154 butler.registry.registerDatasetType(datasetType) 

155 # Load and then put instead of just moving the file in part to ensure 

156 # the version in-repo is written with Python 3 and does not need 

157 # `encoding='latin1'` to be read. 

158 bfKernel = self.getBrighterFatterKernel() 

159 butler.put(bfKernel, datasetType, unboundedDataId) 

160 

161 # The following iterate over the values of the dictionaries returned by the transmission functions 

162 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates, 

163 # which do not give the proper notion of validity. As such unbounded calibration labels are used 

164 # when inserting into the database. In the future these could and probably should be updated to 

165 # properly account for what ranges are considered valid. 

166 

167 # Write optical transmissions 

168 opticsTransmissions = getOpticsTransmission() 

169 datasetType = DatasetType("transmission_optics", 

170 ("instrument", "calibration_label"), 

171 "TransmissionCurve", 

172 universe=butler.registry.dimensions) 

173 butler.registry.registerDatasetType(datasetType) 

174 for entry in opticsTransmissions.values(): 

175 if entry is None: 

176 continue 

177 butler.put(entry, datasetType, unboundedDataId) 

178 

179 # Write transmission sensor 

180 sensorTransmissions = getSensorTransmission() 

181 datasetType = DatasetType("transmission_sensor", 

182 ("instrument", "detector", "calibration_label"), 

183 "TransmissionCurve", 

184 universe=butler.registry.dimensions) 

185 butler.registry.registerDatasetType(datasetType) 

186 for entry in sensorTransmissions.values(): 

187 if entry is None: 

188 continue 

189 for sensor, curve in entry.items(): 

190 dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor) 

191 butler.put(curve, datasetType, dataId) 

192 

193 # Write filter transmissions 

194 filterTransmissions = getFilterTransmission() 

195 datasetType = DatasetType("transmission_filter", 

196 ("instrument", "physical_filter", "calibration_label"), 

197 "TransmissionCurve", 

198 universe=butler.registry.dimensions) 

199 butler.registry.registerDatasetType(datasetType) 

200 for entry in filterTransmissions.values(): 

201 if entry is None: 

202 continue 

203 for band, curve in entry.items(): 

204 dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band) 

205 butler.put(curve, datasetType, dataId) 

206 

207 # Write atmospheric transmissions, this only as dimension of instrument as other areas will only 

208 # look up along this dimension (ISR) 

209 atmosphericTransmissions = getAtmosphereTransmission() 

210 datasetType = DatasetType("transmission_atmosphere", ("instrument",), 

211 "TransmissionCurve", 

212 universe=butler.registry.dimensions) 

213 butler.registry.registerDatasetType(datasetType) 

214 for entry in atmosphericTransmissions.values(): 

215 if entry is None: 

216 continue 

217 butler.put(entry, datasetType, {"instrument": self.getName()}) 

218 

219 # Write defects with validity ranges taken from obs_subaru_data/hsc/defects 

220 # (along with the defects themselves). 

221 datasetType = DatasetType("defects", ("instrument", "detector", "calibration_label"), "Defects", 

222 universe=butler.registry.dimensions) 

223 butler.registry.registerDatasetType(datasetType) 

224 defectPath = os.path.join(getPackageDir("obs_subaru_data"), "hsc", "defects") 

225 camera = self.getCamera() 

226 defectsDict = read_all(defectPath, camera)[0] # This method returns a dict plus the calib type 

227 endOfTime = '20380119T031407' 

228 dimensionRecords = [] 

229 datasetRecords = [] 

230 # First loop just gathers up the things we want to insert, so we 

231 # can do some bulk inserts and minimize the time spent in transaction. 

232 for det in defectsDict: 

233 detector = camera[det] 

234 times = sorted([k for k in defectsDict[det]]) 

235 defects = [defectsDict[det][time] for time in times] 

236 times = times + [parser.parse(endOfTime), ] 

237 for defect, beginTime, endTime in zip(defects, times[:-1], times[1:]): 

238 md = defect.getMetadata() 

239 calibrationLabel = f"defect/{md['CALIBDATE']}/{md['DETECTOR']}" 

240 dataId = DataCoordinate.standardize( 

241 universe=butler.registry.dimensions, 

242 instrument=self.getName(), 

243 calibration_label=calibrationLabel, 

244 detector=detector.getId(), 

245 ) 

246 datasetRecords.append((defect, dataId)) 

247 dimensionRecords.append({ 

248 "instrument": self.getName(), 

249 "name": calibrationLabel, 

250 "datetime_begin": beginTime, 

251 "datetime_end": endTime, 

252 }) 

253 # Second loop actually does the inserts and filesystem writes. 

254 with butler.transaction(): 

255 butler.registry.insertDimensionData("calibration_label", *dimensionRecords) 

256 # TODO: vectorize these puts, once butler APIs for that become 

257 # available. 

258 for defect, dataId in datasetRecords: 

259 butler.put(defect, datasetType, dataId) 

260 

261 def ingestStrayLightData(self, butler, directory, *, transfer=None): 

262 """Ingest externally-produced y-band stray light data files into 

263 a data repository. 

264 

265 Parameters 

266 ---------- 

267 butler : `lsst.daf.butler.Butler` 

268 Butler initialized with the collection to ingest into. 

269 directory : `str` 

270 Directory containing yBackground-*.fits files. 

271 transfer : `str`, optional 

272 If not `None`, must be one of 'move', 'copy', 'hardlink', or 

273 'symlink', indicating how to transfer the files. 

274 """ 

275 calibrationLabel = "y-LED-encoder-on" 

276 # LEDs covered up around 2018-01-01, no need for correctin after that 

277 # date. 

278 datetime_end = datetime.datetime(2018, 1, 1) 

279 datasets = [] 

280 # TODO: should we use a more generic name for the dataset type? 

281 # This is just the (rather HSC-specific) name used in Gen2, and while 

282 # the instances of this dataset are camera-specific, the datasetType 

283 # (which is used in the generic IsrTask) should not be. 

284 datasetType = DatasetType("yBackground", 

285 dimensions=("physical_filter", "detector", "calibration_label"), 

286 storageClass="StrayLightData", 

287 universe=butler.registry.dimensions) 

288 for detector in self.getCamera(): 

289 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits") 

290 if not os.path.exists(path): 

291 log.warn(f"No stray light data found for detector {detector.getId()} @ {path}.") 

292 continue 

293 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(), 

294 "detector": detector.getId(), 

295 "physical_filter": "HSC-Y", 

296 "calibration_label": calibrationLabel}) 

297 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter)) 

298 with butler.transaction(): 

299 butler.registry.registerDatasetType(datasetType) 

300 butler.registry.insertDimensionData("calibration_label", {"instrument": self.getName(), 

301 "name": calibrationLabel, 

302 "datetime_begin": datetime.date.min, 

303 "datetime_end": datetime_end}) 

304 butler.ingest(*datasets, transfer=transfer)