Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_subaru. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Gen3 Butler registry declarations for Hyper Suprime-Cam. 

23""" 

24 

25__all__ = ("HyperSuprimeCam",) 

26 

27import os 

28import pickle 

29import logging 

30 

31from functools import lru_cache 

32 

33import astropy.time 

34from lsst.utils import getPackageDir 

35from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig 

36from lsst.daf.butler import (DatasetType, DataCoordinate, FileDataset, DatasetRef, 

37 CollectionType, Timespan) 

38from lsst.daf.butler.core.utils import getFullTypeName 

39from lsst.obs.base import Instrument, addUnboundedCalibrationLabel 

40from lsst.obs.base.gen2to3 import TranslatorFactory, PhysicalToAbstractFilterKeyHandler 

41 

42from ..hsc.hscPupil import HscPupilFactory 

43from ..hsc.hscFilters import HSC_FILTER_DEFINITIONS 

44from ..hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission, 

45 getFilterTransmission, getAtmosphereTransmission) 

46from .strayLight.formatter import SubaruStrayLightDataFormatter 

47 

48log = logging.getLogger(__name__) 

49 

50 

51class HyperSuprimeCam(Instrument): 

52 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam. 

53 """ 

54 

55 policyName = "hsc" 

56 obsDataPackage = "obs_subaru_data" 

57 filterDefinitions = HSC_FILTER_DEFINITIONS 

58 

59 def __init__(self, **kwargs): 

60 super().__init__(**kwargs) 

61 packageDir = getPackageDir("obs_subaru") 

62 self.configPaths = [os.path.join(packageDir, "config"), 

63 os.path.join(packageDir, "config", self.policyName)] 

64 

65 @classmethod 

66 def getName(cls): 

67 # Docstring inherited from Instrument.getName 

68 return "HSC" 

69 

70 def register(self, registry): 

71 # Docstring inherited from Instrument.register 

72 camera = self.getCamera() 

73 # The maximum values below make Gen3's ObservationDataIdPacker produce 

74 # outputs that match Gen2's ccdExposureId. 

75 obsMax = 21474800 

76 registry.insertDimensionData( 

77 "instrument", 

78 { 

79 "name": self.getName(), 

80 "detector_max": 200, 

81 "visit_max": obsMax, 

82 "exposure_max": obsMax, 

83 "class_name": getFullTypeName(self), 

84 } 

85 ) 

86 registry.insertDimensionData( 

87 "detector", 

88 *[ 

89 { 

90 "instrument": self.getName(), 

91 "id": detector.getId(), 

92 "full_name": detector.getName(), 

93 # TODO: make sure these definitions are consistent with those 

94 # extracted by astro_metadata_translator, and test that they 

95 # remain consistent somehow. 

96 "name_in_raft": detector.getName().split("_")[1], 

97 "raft": detector.getName().split("_")[0], 

98 "purpose": str(detector.getType()).split(".")[-1], 

99 } 

100 for detector in camera 

101 ] 

102 ) 

103 self._registerFilters(registry) 

104 

105 def getRawFormatter(self, dataId): 

106 # Docstring inherited from Instrument.getRawFormatter 

107 # Import the formatter here to prevent a circular dependency. 

108 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter 

109 if dataId["detector"] in (100, 101, 102, 103): 

110 return HyperSuprimeCamCornerRawFormatter 

111 else: 

112 return HyperSuprimeCamRawFormatter 

113 

114 def getCamera(self): 

115 """Retrieve the cameraGeom representation of HSC. 

116 

117 This is a temporary API that should go away once obs_ packages have 

118 a standardized approach to writing versioned cameras to a Gen3 repo. 

119 """ 

120 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "camera") 

121 return self._getCameraFromPath(path) 

122 

123 @staticmethod 

124 @lru_cache() 

125 def _getCameraFromPath(path): 

126 """Return the camera geometry given solely the path to the location 

127 of that definition.""" 

128 config = CameraConfig() 

129 config.load(os.path.join(path, "camera.py")) 

130 return makeCameraFromPath( 

131 cameraConfig=config, 

132 ampInfoPath=path, 

133 shortNameFunc=lambda name: name.replace(" ", "_"), 

134 pupilFactoryClass=HscPupilFactory 

135 ) 

136 

137 def getBrighterFatterKernel(self): 

138 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`. 

139 

140 This is a temporary API that should go away once obs_ packages have 

141 a standardized approach to writing versioned kernels to a Gen3 repo. 

142 """ 

143 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "brighter_fatter_kernel.pkl") 

144 with open(path, "rb") as fd: 

145 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2 

146 return kernel 

147 

148 def writeAdditionalCuratedCalibrations(self, butler, run=None): 

149 # Get an unbounded calibration label 

150 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

151 

152 # Write brighter-fatter kernel, with an infinite validity range. 

153 datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray", 

154 universe=butler.registry.dimensions) 

155 butler.registry.registerDatasetType(datasetType) 

156 # Load and then put instead of just moving the file in part to ensure 

157 # the version in-repo is written with Python 3 and does not need 

158 # `encoding='latin1'` to be read. 

159 bfKernel = self.getBrighterFatterKernel() 

160 butler.put(bfKernel, datasetType, unboundedDataId, run=run) 

161 

162 # The following iterate over the values of the dictionaries returned by the transmission functions 

163 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates, 

164 # which do not give the proper notion of validity. As such unbounded calibration labels are used 

165 # when inserting into the database. In the future these could and probably should be updated to 

166 # properly account for what ranges are considered valid. 

167 

168 # Write optical transmissions 

169 opticsTransmissions = getOpticsTransmission() 

170 datasetType = DatasetType("transmission_optics", 

171 ("instrument", "calibration_label"), 

172 "TransmissionCurve", 

173 universe=butler.registry.dimensions) 

174 butler.registry.registerDatasetType(datasetType) 

175 for entry in opticsTransmissions.values(): 

176 if entry is None: 

177 continue 

178 butler.put(entry, datasetType, unboundedDataId, run=run) 

179 

180 # Write transmission sensor 

181 sensorTransmissions = getSensorTransmission() 

182 datasetType = DatasetType("transmission_sensor", 

183 ("instrument", "detector", "calibration_label"), 

184 "TransmissionCurve", 

185 universe=butler.registry.dimensions) 

186 butler.registry.registerDatasetType(datasetType) 

187 for entry in sensorTransmissions.values(): 

188 if entry is None: 

189 continue 

190 for sensor, curve in entry.items(): 

191 dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor) 

192 butler.put(curve, datasetType, dataId, run=run) 

193 

194 # Write filter transmissions 

195 filterTransmissions = getFilterTransmission() 

196 datasetType = DatasetType("transmission_filter", 

197 ("instrument", "physical_filter", "calibration_label"), 

198 "TransmissionCurve", 

199 universe=butler.registry.dimensions) 

200 butler.registry.registerDatasetType(datasetType) 

201 for entry in filterTransmissions.values(): 

202 if entry is None: 

203 continue 

204 for band, curve in entry.items(): 

205 dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band) 

206 butler.put(curve, datasetType, dataId, run=run) 

207 

208 # Write atmospheric transmissions, this only as dimension of instrument as other areas will only 

209 # look up along this dimension (ISR) 

210 atmosphericTransmissions = getAtmosphereTransmission() 

211 datasetType = DatasetType("transmission_atmosphere", ("instrument",), 

212 "TransmissionCurve", 

213 universe=butler.registry.dimensions) 

214 butler.registry.registerDatasetType(datasetType) 

215 for entry in atmosphericTransmissions.values(): 

216 if entry is None: 

217 continue 

218 butler.put(entry, datasetType, {"instrument": self.getName()}, run=run) 

219 

220 def ingestStrayLightData(self, butler, directory, *, transfer=None, run=None): 

221 """Ingest externally-produced y-band stray light data files into 

222 a data repository. 

223 

224 Parameters 

225 ---------- 

226 butler : `lsst.daf.butler.Butler` 

227 Butler initialized with the collection to ingest into. 

228 directory : `str` 

229 Directory containing yBackground-*.fits files. 

230 transfer : `str`, optional 

231 If not `None`, must be one of 'move', 'copy', 'hardlink', or 

232 'symlink', indicating how to transfer the files. 

233 run : `str` 

234 Run to use for this collection of calibrations. If `None` the 

235 collection name is worked out automatically from the instrument 

236 name and other metadata. 

237 """ 

238 if run is None: 

239 run = self.makeCollectionName("calib") 

240 butler.registry.registerCollection(run, type=CollectionType.RUN) 

241 

242 calibrationLabel = "y-LED-encoder-on" 

243 # LEDs covered up around 2018-01-01, no need for correctin after that 

244 # date. 

245 timespan = Timespan(begin=None, end=astropy.time.Time("2018-01-01", format="iso", scale="tai")) 

246 datasets = [] 

247 # TODO: should we use a more generic name for the dataset type? 

248 # This is just the (rather HSC-specific) name used in Gen2, and while 

249 # the instances of this dataset are camera-specific, the datasetType 

250 # (which is used in the generic IsrTask) should not be. 

251 datasetType = DatasetType("yBackground", 

252 dimensions=("physical_filter", "detector", "calibration_label"), 

253 storageClass="StrayLightData", 

254 universe=butler.registry.dimensions) 

255 for detector in self.getCamera(): 

256 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits") 

257 if not os.path.exists(path): 

258 log.warning(f"No stray light data found for detector {detector.getId()} @ {path}.") 

259 continue 

260 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(), 

261 "detector": detector.getId(), 

262 "physical_filter": "HSC-Y", 

263 "calibration_label": calibrationLabel}) 

264 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter)) 

265 butler.registry.registerDatasetType(datasetType) 

266 with butler.transaction(): 

267 butler.registry.insertDimensionData("calibration_label", {"instrument": self.getName(), 

268 "name": calibrationLabel, 

269 "timespan": timespan}) 

270 butler.ingest(*datasets, transfer=transfer, run=run) 

271 

272 def makeDataIdTranslatorFactory(self) -> TranslatorFactory: 

273 # Docstring inherited from lsst.obs.base.Instrument. 

274 factory = TranslatorFactory() 

275 factory.addGenericInstrumentRules(self.getName()) 

276 # Translate Gen2 `filter` to abstract_filter if it hasn't been consumed 

277 # yet and gen2keys includes tract. 

278 factory.addRule(PhysicalToAbstractFilterKeyHandler(self.filterDefinitions), 

279 instrument=self.getName(), gen2keys=("filter", "tract"), consume=("filter",)) 

280 return factory