Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_subaru. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Gen3 Butler registry declarations for Hyper Suprime-Cam. 

23""" 

24 

25__all__ = ("HyperSuprimeCam",) 

26 

27import os 

28import pickle 

29import logging 

30 

31import astropy.time 

32from lsst.utils import getPackageDir 

33from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig 

34from lsst.daf.butler import (DatasetType, DataCoordinate, FileDataset, DatasetRef, 

35 TIMESPAN_MIN, TIMESPAN_MAX) 

36from lsst.obs.base import Instrument, addUnboundedCalibrationLabel 

37from lsst.pipe.tasks.read_curated_calibs import read_all 

38 

39from ..hsc.hscPupil import HscPupilFactory 

40from ..hsc.hscFilters import HSC_FILTER_DEFINITIONS 

41from ..hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission, 

42 getFilterTransmission, getAtmosphereTransmission) 

43from .strayLight.formatter import SubaruStrayLightDataFormatter 

44 

45log = logging.getLogger(__name__) 

46 

47 

48class HyperSuprimeCam(Instrument): 

49 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam. 

50 """ 

51 

52 filterDefinitions = HSC_FILTER_DEFINITIONS 

53 

54 def __init__(self, **kwargs): 

55 super().__init__(**kwargs) 

56 packageDir = getPackageDir("obs_subaru") 

57 self.configPaths = [os.path.join(packageDir, "config"), 

58 os.path.join(packageDir, "config", "hsc")] 

59 

60 @classmethod 

61 def getName(cls): 

62 # Docstring inherited from Instrument.getName 

63 return "HSC" 

64 

65 def register(self, registry): 

66 # Docstring inherited from Instrument.register 

67 camera = self.getCamera() 

68 # The maximum values below make Gen3's ObservationDataIdPacker produce 

69 # outputs that match Gen2's ccdExposureId. 

70 obsMax = 21474800 

71 registry.insertDimensionData( 

72 "instrument", 

73 { 

74 "name": self.getName(), 

75 "detector_max": 200, 

76 "visit_max": obsMax, 

77 "exposure_max": obsMax 

78 } 

79 ) 

80 registry.insertDimensionData( 

81 "detector", 

82 *[ 

83 { 

84 "instrument": self.getName(), 

85 "id": detector.getId(), 

86 "full_name": detector.getName(), 

87 # TODO: make sure these definitions are consistent with those 

88 # extracted by astro_metadata_translator, and test that they 

89 # remain consistent somehow. 

90 "name_in_raft": detector.getName().split("_")[1], 

91 "raft": detector.getName().split("_")[0], 

92 "purpose": str(detector.getType()).split(".")[-1], 

93 } 

94 for detector in camera 

95 ] 

96 ) 

97 self._registerFilters(registry) 

98 

99 def getRawFormatter(self, dataId): 

100 # Docstring inherited from Instrument.getRawFormatter 

101 # Import the formatter here to prevent a circular dependency. 

102 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter 

103 if dataId["detector"] in (100, 101, 102, 103): 

104 return HyperSuprimeCamCornerRawFormatter 

105 else: 

106 return HyperSuprimeCamRawFormatter 

107 

108 def getCamera(self): 

109 """Retrieve the cameraGeom representation of HSC. 

110 

111 This is a temporary API that should go away once obs_ packages have 

112 a standardized approach to writing versioned cameras to a Gen3 repo. 

113 """ 

114 path = os.path.join(getPackageDir("obs_subaru"), "hsc", "camera") 

115 config = CameraConfig() 

116 config.load(os.path.join(path, "camera.py")) 

117 return makeCameraFromPath( 

118 cameraConfig=config, 

119 ampInfoPath=path, 

120 shortNameFunc=lambda name: name.replace(" ", "_"), 

121 pupilFactoryClass=HscPupilFactory 

122 ) 

123 

124 def getBrighterFatterKernel(self): 

125 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`. 

126 

127 This is a temporary API that should go away once obs_ packages have 

128 a standardized approach to writing versioned kernels to a Gen3 repo. 

129 """ 

130 path = os.path.join(getPackageDir("obs_subaru"), "hsc", "brighter_fatter_kernel.pkl") 

131 with open(path, "rb") as fd: 

132 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2 

133 return kernel 

134 

135 def writeCuratedCalibrations(self, butler): 

136 """Write human-curated calibration Datasets to the given Butler with 

137 the appropriate validity ranges. 

138 

139 This is a temporary API that should go away once obs_ packages have 

140 a standardized approach to this problem. 

141 """ 

142 

143 # Write cameraGeom.Camera, with an infinite validity range. 

144 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

145 universe=butler.registry.dimensions) 

146 butler.registry.registerDatasetType(datasetType) 

147 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

148 camera = self.getCamera() 

149 butler.put(camera, datasetType, unboundedDataId) 

150 

151 # Write brighter-fatter kernel, with an infinite validity range. 

152 datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray", 

153 universe=butler.registry.dimensions) 

154 butler.registry.registerDatasetType(datasetType) 

155 # Load and then put instead of just moving the file in part to ensure 

156 # the version in-repo is written with Python 3 and does not need 

157 # `encoding='latin1'` to be read. 

158 bfKernel = self.getBrighterFatterKernel() 

159 butler.put(bfKernel, datasetType, unboundedDataId) 

160 

161 # The following iterate over the values of the dictionaries returned by the transmission functions 

162 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates, 

163 # which do not give the proper notion of validity. As such unbounded calibration labels are used 

164 # when inserting into the database. In the future these could and probably should be updated to 

165 # properly account for what ranges are considered valid. 

166 

167 # Write optical transmissions 

168 opticsTransmissions = getOpticsTransmission() 

169 datasetType = DatasetType("transmission_optics", 

170 ("instrument", "calibration_label"), 

171 "TransmissionCurve", 

172 universe=butler.registry.dimensions) 

173 butler.registry.registerDatasetType(datasetType) 

174 for entry in opticsTransmissions.values(): 

175 if entry is None: 

176 continue 

177 butler.put(entry, datasetType, unboundedDataId) 

178 

179 # Write transmission sensor 

180 sensorTransmissions = getSensorTransmission() 

181 datasetType = DatasetType("transmission_sensor", 

182 ("instrument", "detector", "calibration_label"), 

183 "TransmissionCurve", 

184 universe=butler.registry.dimensions) 

185 butler.registry.registerDatasetType(datasetType) 

186 for entry in sensorTransmissions.values(): 

187 if entry is None: 

188 continue 

189 for sensor, curve in entry.items(): 

190 dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor) 

191 butler.put(curve, datasetType, dataId) 

192 

193 # Write filter transmissions 

194 filterTransmissions = getFilterTransmission() 

195 datasetType = DatasetType("transmission_filter", 

196 ("instrument", "physical_filter", "calibration_label"), 

197 "TransmissionCurve", 

198 universe=butler.registry.dimensions) 

199 butler.registry.registerDatasetType(datasetType) 

200 for entry in filterTransmissions.values(): 

201 if entry is None: 

202 continue 

203 for band, curve in entry.items(): 

204 dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band) 

205 butler.put(curve, datasetType, dataId) 

206 

207 # Write atmospheric transmissions, this only as dimension of instrument as other areas will only 

208 # look up along this dimension (ISR) 

209 atmosphericTransmissions = getAtmosphereTransmission() 

210 datasetType = DatasetType("transmission_atmosphere", ("instrument",), 

211 "TransmissionCurve", 

212 universe=butler.registry.dimensions) 

213 butler.registry.registerDatasetType(datasetType) 

214 for entry in atmosphericTransmissions.values(): 

215 if entry is None: 

216 continue 

217 butler.put(entry, datasetType, {"instrument": self.getName()}) 

218 

219 # Write defects with validity ranges taken from obs_subaru_data/hsc/defects 

220 # (along with the defects themselves). 

221 datasetType = DatasetType("defects", ("instrument", "detector", "calibration_label"), "Defects", 

222 universe=butler.registry.dimensions) 

223 butler.registry.registerDatasetType(datasetType) 

224 defectPath = os.path.join(getPackageDir("obs_subaru_data"), "hsc", "defects") 

225 camera = self.getCamera() 

226 defectsDict = read_all(defectPath, camera)[0] # This method returns a dict plus the calib type 

227 endOfTime = TIMESPAN_MAX 

228 dimensionRecords = [] 

229 datasetRecords = [] 

230 # First loop just gathers up the things we want to insert, so we 

231 # can do some bulk inserts and minimize the time spent in transaction. 

232 for det in defectsDict: 

233 detector = camera[det] 

234 times = sorted([k for k in defectsDict[det]]) 

235 defects = [defectsDict[det][time] for time in times] 

236 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times] 

237 times += [endOfTime] 

238 for defect, beginTime, endTime in zip(defects, times[:-1], times[1:]): 

239 md = defect.getMetadata() 

240 calibrationLabel = f"defect/{md['CALIBDATE']}/{md['DETECTOR']}" 

241 dataId = DataCoordinate.standardize( 

242 universe=butler.registry.dimensions, 

243 instrument=self.getName(), 

244 calibration_label=calibrationLabel, 

245 detector=detector.getId(), 

246 ) 

247 datasetRecords.append((defect, dataId)) 

248 dimensionRecords.append({ 

249 "instrument": self.getName(), 

250 "name": calibrationLabel, 

251 "datetime_begin": beginTime, 

252 "datetime_end": endTime, 

253 }) 

254 # Second loop actually does the inserts and filesystem writes. 

255 with butler.transaction(): 

256 butler.registry.insertDimensionData("calibration_label", *dimensionRecords) 

257 # TODO: vectorize these puts, once butler APIs for that become 

258 # available. 

259 for defect, dataId in datasetRecords: 

260 butler.put(defect, datasetType, dataId) 

261 

262 def ingestStrayLightData(self, butler, directory, *, transfer=None): 

263 """Ingest externally-produced y-band stray light data files into 

264 a data repository. 

265 

266 Parameters 

267 ---------- 

268 butler : `lsst.daf.butler.Butler` 

269 Butler initialized with the collection to ingest into. 

270 directory : `str` 

271 Directory containing yBackground-*.fits files. 

272 transfer : `str`, optional 

273 If not `None`, must be one of 'move', 'copy', 'hardlink', or 

274 'symlink', indicating how to transfer the files. 

275 """ 

276 calibrationLabel = "y-LED-encoder-on" 

277 # LEDs covered up around 2018-01-01, no need for correctin after that 

278 # date. 

279 datetime_begin = TIMESPAN_MIN 

280 datetime_end = astropy.time.Time("2018-01-01", format="iso", scale="tai") 

281 datasets = [] 

282 # TODO: should we use a more generic name for the dataset type? 

283 # This is just the (rather HSC-specific) name used in Gen2, and while 

284 # the instances of this dataset are camera-specific, the datasetType 

285 # (which is used in the generic IsrTask) should not be. 

286 datasetType = DatasetType("yBackground", 

287 dimensions=("physical_filter", "detector", "calibration_label"), 

288 storageClass="StrayLightData", 

289 universe=butler.registry.dimensions) 

290 for detector in self.getCamera(): 

291 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits") 

292 if not os.path.exists(path): 

293 log.warn(f"No stray light data found for detector {detector.getId()} @ {path}.") 

294 continue 

295 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(), 

296 "detector": detector.getId(), 

297 "physical_filter": "HSC-Y", 

298 "calibration_label": calibrationLabel}) 

299 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter)) 

300 with butler.transaction(): 

301 butler.registry.registerDatasetType(datasetType) 

302 butler.registry.insertDimensionData("calibration_label", {"instrument": self.getName(), 

303 "name": calibrationLabel, 

304 "datetime_begin": datetime_begin, 

305 "datetime_end": datetime_end}) 

306 butler.ingest(*datasets, transfer=transfer)