Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_subaru. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Gen3 Butler registry declarations for Hyper Suprime-Cam. 

23""" 

24 

25__all__ = ("HyperSuprimeCam",) 

26 

27import os 

28import pickle 

29import logging 

30 

31import astropy.time 

32from lsst.utils import getPackageDir 

33from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig 

34from lsst.daf.butler import (DatasetType, DataCoordinate, FileDataset, DatasetRef, 

35 TIMESPAN_MIN) 

36from lsst.obs.base import Instrument, addUnboundedCalibrationLabel 

37 

38from ..hsc.hscPupil import HscPupilFactory 

39from ..hsc.hscFilters import HSC_FILTER_DEFINITIONS 

40from ..hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission, 

41 getFilterTransmission, getAtmosphereTransmission) 

42from .strayLight.formatter import SubaruStrayLightDataFormatter 

43 

44log = logging.getLogger(__name__) 

45 

46 

47class HyperSuprimeCam(Instrument): 

48 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam. 

49 """ 

50 

51 policyName = "hsc" 

52 obsDataPackage = "obs_subaru_data" 

53 filterDefinitions = HSC_FILTER_DEFINITIONS 

54 

55 def __init__(self, **kwargs): 

56 super().__init__(**kwargs) 

57 packageDir = getPackageDir("obs_subaru") 

58 self.configPaths = [os.path.join(packageDir, "config"), 

59 os.path.join(packageDir, "config", self.policyName)] 

60 

61 @classmethod 

62 def getName(cls): 

63 # Docstring inherited from Instrument.getName 

64 return "HSC" 

65 

66 def register(self, registry): 

67 # Docstring inherited from Instrument.register 

68 camera = self.getCamera() 

69 # The maximum values below make Gen3's ObservationDataIdPacker produce 

70 # outputs that match Gen2's ccdExposureId. 

71 obsMax = 21474800 

72 registry.insertDimensionData( 

73 "instrument", 

74 { 

75 "name": self.getName(), 

76 "detector_max": 200, 

77 "visit_max": obsMax, 

78 "exposure_max": obsMax 

79 } 

80 ) 

81 registry.insertDimensionData( 

82 "detector", 

83 *[ 

84 { 

85 "instrument": self.getName(), 

86 "id": detector.getId(), 

87 "full_name": detector.getName(), 

88 # TODO: make sure these definitions are consistent with those 

89 # extracted by astro_metadata_translator, and test that they 

90 # remain consistent somehow. 

91 "name_in_raft": detector.getName().split("_")[1], 

92 "raft": detector.getName().split("_")[0], 

93 "purpose": str(detector.getType()).split(".")[-1], 

94 } 

95 for detector in camera 

96 ] 

97 ) 

98 self._registerFilters(registry) 

99 

100 def getRawFormatter(self, dataId): 

101 # Docstring inherited from Instrument.getRawFormatter 

102 # Import the formatter here to prevent a circular dependency. 

103 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter 

104 if dataId["detector"] in (100, 101, 102, 103): 

105 return HyperSuprimeCamCornerRawFormatter 

106 else: 

107 return HyperSuprimeCamRawFormatter 

108 

109 def getCamera(self): 

110 """Retrieve the cameraGeom representation of HSC. 

111 

112 This is a temporary API that should go away once obs_ packages have 

113 a standardized approach to writing versioned cameras to a Gen3 repo. 

114 """ 

115 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "camera") 

116 config = CameraConfig() 

117 config.load(os.path.join(path, "camera.py")) 

118 return makeCameraFromPath( 

119 cameraConfig=config, 

120 ampInfoPath=path, 

121 shortNameFunc=lambda name: name.replace(" ", "_"), 

122 pupilFactoryClass=HscPupilFactory 

123 ) 

124 

125 def getBrighterFatterKernel(self): 

126 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`. 

127 

128 This is a temporary API that should go away once obs_ packages have 

129 a standardized approach to writing versioned kernels to a Gen3 repo. 

130 """ 

131 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "brighter_fatter_kernel.pkl") 

132 with open(path, "rb") as fd: 

133 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2 

134 return kernel 

135 

136 def writeCuratedCalibrations(self, butler): 

137 """Write human-curated calibration Datasets to the given Butler with 

138 the appropriate validity ranges. 

139 

140 This is a temporary API that should go away once obs_ packages have 

141 a standardized approach to this problem. 

142 """ 

143 

144 # Write the generic calibrations supported by all instruments 

145 super().writeCuratedCalibrations(butler) 

146 

147 # Get an unbounded calibration label 

148 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

149 

150 # Write brighter-fatter kernel, with an infinite validity range. 

151 datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray", 

152 universe=butler.registry.dimensions) 

153 butler.registry.registerDatasetType(datasetType) 

154 # Load and then put instead of just moving the file in part to ensure 

155 # the version in-repo is written with Python 3 and does not need 

156 # `encoding='latin1'` to be read. 

157 bfKernel = self.getBrighterFatterKernel() 

158 butler.put(bfKernel, datasetType, unboundedDataId) 

159 

160 # The following iterate over the values of the dictionaries returned by the transmission functions 

161 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates, 

162 # which do not give the proper notion of validity. As such unbounded calibration labels are used 

163 # when inserting into the database. In the future these could and probably should be updated to 

164 # properly account for what ranges are considered valid. 

165 

166 # Write optical transmissions 

167 opticsTransmissions = getOpticsTransmission() 

168 datasetType = DatasetType("transmission_optics", 

169 ("instrument", "calibration_label"), 

170 "TransmissionCurve", 

171 universe=butler.registry.dimensions) 

172 butler.registry.registerDatasetType(datasetType) 

173 for entry in opticsTransmissions.values(): 

174 if entry is None: 

175 continue 

176 butler.put(entry, datasetType, unboundedDataId) 

177 

178 # Write transmission sensor 

179 sensorTransmissions = getSensorTransmission() 

180 datasetType = DatasetType("transmission_sensor", 

181 ("instrument", "detector", "calibration_label"), 

182 "TransmissionCurve", 

183 universe=butler.registry.dimensions) 

184 butler.registry.registerDatasetType(datasetType) 

185 for entry in sensorTransmissions.values(): 

186 if entry is None: 

187 continue 

188 for sensor, curve in entry.items(): 

189 dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor) 

190 butler.put(curve, datasetType, dataId) 

191 

192 # Write filter transmissions 

193 filterTransmissions = getFilterTransmission() 

194 datasetType = DatasetType("transmission_filter", 

195 ("instrument", "physical_filter", "calibration_label"), 

196 "TransmissionCurve", 

197 universe=butler.registry.dimensions) 

198 butler.registry.registerDatasetType(datasetType) 

199 for entry in filterTransmissions.values(): 

200 if entry is None: 

201 continue 

202 for band, curve in entry.items(): 

203 dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band) 

204 butler.put(curve, datasetType, dataId) 

205 

206 # Write atmospheric transmissions, this only as dimension of instrument as other areas will only 

207 # look up along this dimension (ISR) 

208 atmosphericTransmissions = getAtmosphereTransmission() 

209 datasetType = DatasetType("transmission_atmosphere", ("instrument",), 

210 "TransmissionCurve", 

211 universe=butler.registry.dimensions) 

212 butler.registry.registerDatasetType(datasetType) 

213 for entry in atmosphericTransmissions.values(): 

214 if entry is None: 

215 continue 

216 butler.put(entry, datasetType, {"instrument": self.getName()}) 

217 

218 def ingestStrayLightData(self, butler, directory, *, transfer=None): 

219 """Ingest externally-produced y-band stray light data files into 

220 a data repository. 

221 

222 Parameters 

223 ---------- 

224 butler : `lsst.daf.butler.Butler` 

225 Butler initialized with the collection to ingest into. 

226 directory : `str` 

227 Directory containing yBackground-*.fits files. 

228 transfer : `str`, optional 

229 If not `None`, must be one of 'move', 'copy', 'hardlink', or 

230 'symlink', indicating how to transfer the files. 

231 """ 

232 calibrationLabel = "y-LED-encoder-on" 

233 # LEDs covered up around 2018-01-01, no need for correctin after that 

234 # date. 

235 datetime_begin = TIMESPAN_MIN 

236 datetime_end = astropy.time.Time("2018-01-01", format="iso", scale="tai") 

237 datasets = [] 

238 # TODO: should we use a more generic name for the dataset type? 

239 # This is just the (rather HSC-specific) name used in Gen2, and while 

240 # the instances of this dataset are camera-specific, the datasetType 

241 # (which is used in the generic IsrTask) should not be. 

242 datasetType = DatasetType("yBackground", 

243 dimensions=("physical_filter", "detector", "calibration_label"), 

244 storageClass="StrayLightData", 

245 universe=butler.registry.dimensions) 

246 for detector in self.getCamera(): 

247 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits") 

248 if not os.path.exists(path): 

249 log.warn(f"No stray light data found for detector {detector.getId()} @ {path}.") 

250 continue 

251 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(), 

252 "detector": detector.getId(), 

253 "physical_filter": "HSC-Y", 

254 "calibration_label": calibrationLabel}) 

255 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter)) 

256 with butler.transaction(): 

257 butler.registry.registerDatasetType(datasetType) 

258 butler.registry.insertDimensionData("calibration_label", {"instrument": self.getName(), 

259 "name": calibrationLabel, 

260 "datetime_begin": datetime_begin, 

261 "datetime_end": datetime_end}) 

262 butler.ingest(*datasets, transfer=transfer)