Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_subaru. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Gen3 Butler registry declarations for Hyper Suprime-Cam. 

23""" 

24 

25__all__ = ("HyperSuprimeCam",) 

26 

27import os 

28import pickle 

29import logging 

30 

31from functools import lru_cache 

32 

33import astropy.time 

34from lsst.utils import getPackageDir 

35from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig 

36from lsst.daf.butler import (DatasetType, DataCoordinate, FileDataset, DatasetRef, 

37 TIMESPAN_MIN, CollectionType) 

38from lsst.daf.butler.core.utils import getFullTypeName 

39from lsst.obs.base import Instrument, addUnboundedCalibrationLabel 

40from lsst.obs.base.gen2to3 import TranslatorFactory, PhysicalToAbstractFilterKeyHandler 

41 

42from ..hsc.hscPupil import HscPupilFactory 

43from ..hsc.hscFilters import HSC_FILTER_DEFINITIONS 

44from ..hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission, 

45 getFilterTransmission, getAtmosphereTransmission) 

46from .strayLight.formatter import SubaruStrayLightDataFormatter 

47 

48log = logging.getLogger(__name__) 

49 

50 

51class HyperSuprimeCam(Instrument): 

52 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam. 

53 """ 

54 

55 policyName = "hsc" 

56 obsDataPackage = "obs_subaru_data" 

57 filterDefinitions = HSC_FILTER_DEFINITIONS 

58 

59 def __init__(self, **kwargs): 

60 super().__init__(**kwargs) 

61 packageDir = getPackageDir("obs_subaru") 

62 self.configPaths = [os.path.join(packageDir, "config"), 

63 os.path.join(packageDir, "config", self.policyName)] 

64 

65 @classmethod 

66 def getName(cls): 

67 # Docstring inherited from Instrument.getName 

68 return "HSC" 

69 

70 def register(self, registry): 

71 # Docstring inherited from Instrument.register 

72 camera = self.getCamera() 

73 # The maximum values below make Gen3's ObservationDataIdPacker produce 

74 # outputs that match Gen2's ccdExposureId. 

75 obsMax = 21474800 

76 registry.insertDimensionData( 

77 "instrument", 

78 { 

79 "name": self.getName(), 

80 "detector_max": 200, 

81 "visit_max": obsMax, 

82 "exposure_max": obsMax, 

83 "class_name": getFullTypeName(self), 

84 } 

85 ) 

86 registry.insertDimensionData( 

87 "detector", 

88 *[ 

89 { 

90 "instrument": self.getName(), 

91 "id": detector.getId(), 

92 "full_name": detector.getName(), 

93 # TODO: make sure these definitions are consistent with those 

94 # extracted by astro_metadata_translator, and test that they 

95 # remain consistent somehow. 

96 "name_in_raft": detector.getName().split("_")[1], 

97 "raft": detector.getName().split("_")[0], 

98 "purpose": str(detector.getType()).split(".")[-1], 

99 } 

100 for detector in camera 

101 ] 

102 ) 

103 self._registerFilters(registry) 

104 

105 def getRawFormatter(self, dataId): 

106 # Docstring inherited from Instrument.getRawFormatter 

107 # Import the formatter here to prevent a circular dependency. 

108 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter 

109 if dataId["detector"] in (100, 101, 102, 103): 

110 return HyperSuprimeCamCornerRawFormatter 

111 else: 

112 return HyperSuprimeCamRawFormatter 

113 

114 def getCamera(self): 

115 """Retrieve the cameraGeom representation of HSC. 

116 

117 This is a temporary API that should go away once obs_ packages have 

118 a standardized approach to writing versioned cameras to a Gen3 repo. 

119 """ 

120 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "camera") 

121 return self._getCameraFromPath(path) 

122 

123 @staticmethod 

124 @lru_cache() 

125 def _getCameraFromPath(path): 

126 """Return the camera geometry given solely the path to the location 

127 of that definition.""" 

128 config = CameraConfig() 

129 config.load(os.path.join(path, "camera.py")) 

130 return makeCameraFromPath( 

131 cameraConfig=config, 

132 ampInfoPath=path, 

133 shortNameFunc=lambda name: name.replace(" ", "_"), 

134 pupilFactoryClass=HscPupilFactory 

135 ) 

136 

137 def getBrighterFatterKernel(self): 

138 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`. 

139 

140 This is a temporary API that should go away once obs_ packages have 

141 a standardized approach to writing versioned kernels to a Gen3 repo. 

142 """ 

143 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "brighter_fatter_kernel.pkl") 

144 with open(path, "rb") as fd: 

145 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2 

146 return kernel 

147 

148 def writeAdditionalCuratedCalibrations(self, butler, run=None): 

149 # Get an unbounded calibration label 

150 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

151 

152 # Write brighter-fatter kernel, with an infinite validity range. 

153 datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray", 

154 universe=butler.registry.dimensions) 

155 butler.registry.registerDatasetType(datasetType) 

156 # Load and then put instead of just moving the file in part to ensure 

157 # the version in-repo is written with Python 3 and does not need 

158 # `encoding='latin1'` to be read. 

159 bfKernel = self.getBrighterFatterKernel() 

160 butler.put(bfKernel, datasetType, unboundedDataId, run=run) 

161 

162 # The following iterate over the values of the dictionaries returned by the transmission functions 

163 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates, 

164 # which do not give the proper notion of validity. As such unbounded calibration labels are used 

165 # when inserting into the database. In the future these could and probably should be updated to 

166 # properly account for what ranges are considered valid. 

167 

168 # Write optical transmissions 

169 opticsTransmissions = getOpticsTransmission() 

170 datasetType = DatasetType("transmission_optics", 

171 ("instrument", "calibration_label"), 

172 "TransmissionCurve", 

173 universe=butler.registry.dimensions) 

174 butler.registry.registerDatasetType(datasetType) 

175 for entry in opticsTransmissions.values(): 

176 if entry is None: 

177 continue 

178 butler.put(entry, datasetType, unboundedDataId, run=run) 

179 

180 # Write transmission sensor 

181 sensorTransmissions = getSensorTransmission() 

182 datasetType = DatasetType("transmission_sensor", 

183 ("instrument", "detector", "calibration_label"), 

184 "TransmissionCurve", 

185 universe=butler.registry.dimensions) 

186 butler.registry.registerDatasetType(datasetType) 

187 for entry in sensorTransmissions.values(): 

188 if entry is None: 

189 continue 

190 for sensor, curve in entry.items(): 

191 dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor) 

192 butler.put(curve, datasetType, dataId, run=run) 

193 

194 # Write filter transmissions 

195 filterTransmissions = getFilterTransmission() 

196 datasetType = DatasetType("transmission_filter", 

197 ("instrument", "physical_filter", "calibration_label"), 

198 "TransmissionCurve", 

199 universe=butler.registry.dimensions) 

200 butler.registry.registerDatasetType(datasetType) 

201 for entry in filterTransmissions.values(): 

202 if entry is None: 

203 continue 

204 for band, curve in entry.items(): 

205 dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band) 

206 butler.put(curve, datasetType, dataId, run=run) 

207 

208 # Write atmospheric transmissions, this only as dimension of instrument as other areas will only 

209 # look up along this dimension (ISR) 

210 atmosphericTransmissions = getAtmosphereTransmission() 

211 datasetType = DatasetType("transmission_atmosphere", ("instrument",), 

212 "TransmissionCurve", 

213 universe=butler.registry.dimensions) 

214 butler.registry.registerDatasetType(datasetType) 

215 for entry in atmosphericTransmissions.values(): 

216 if entry is None: 

217 continue 

218 butler.put(entry, datasetType, {"instrument": self.getName()}, run=run) 

219 

220 def ingestStrayLightData(self, butler, directory, *, transfer=None, run=None): 

221 """Ingest externally-produced y-band stray light data files into 

222 a data repository. 

223 

224 Parameters 

225 ---------- 

226 butler : `lsst.daf.butler.Butler` 

227 Butler initialized with the collection to ingest into. 

228 directory : `str` 

229 Directory containing yBackground-*.fits files. 

230 transfer : `str`, optional 

231 If not `None`, must be one of 'move', 'copy', 'hardlink', or 

232 'symlink', indicating how to transfer the files. 

233 run : `str` 

234 Run to use for this collection of calibrations. If `None` the 

235 collection name is worked out automatically from the instrument 

236 name and other metadata. 

237 """ 

238 if run is None: 

239 run = self.makeCollectionName("calib") 

240 butler.registry.registerCollection(run, type=CollectionType.RUN) 

241 

242 calibrationLabel = "y-LED-encoder-on" 

243 # LEDs covered up around 2018-01-01, no need for correctin after that 

244 # date. 

245 datetime_begin = TIMESPAN_MIN 

246 datetime_end = astropy.time.Time("2018-01-01", format="iso", scale="tai") 

247 datasets = [] 

248 # TODO: should we use a more generic name for the dataset type? 

249 # This is just the (rather HSC-specific) name used in Gen2, and while 

250 # the instances of this dataset are camera-specific, the datasetType 

251 # (which is used in the generic IsrTask) should not be. 

252 datasetType = DatasetType("yBackground", 

253 dimensions=("physical_filter", "detector", "calibration_label"), 

254 storageClass="StrayLightData", 

255 universe=butler.registry.dimensions) 

256 for detector in self.getCamera(): 

257 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits") 

258 if not os.path.exists(path): 

259 log.warning(f"No stray light data found for detector {detector.getId()} @ {path}.") 

260 continue 

261 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(), 

262 "detector": detector.getId(), 

263 "physical_filter": "HSC-Y", 

264 "calibration_label": calibrationLabel}) 

265 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter)) 

266 butler.registry.registerDatasetType(datasetType) 

267 with butler.transaction(): 

268 butler.registry.insertDimensionData("calibration_label", {"instrument": self.getName(), 

269 "name": calibrationLabel, 

270 "datetime_begin": datetime_begin, 

271 "datetime_end": datetime_end}) 

272 butler.ingest(*datasets, transfer=transfer, run=run) 

273 

274 def makeDataIdTranslatorFactory(self) -> TranslatorFactory: 

275 # Docstring inherited from lsst.obs.base.Instrument. 

276 factory = TranslatorFactory() 

277 factory.addGenericInstrumentRules(self.getName()) 

278 # Translate Gen2 `filter` to abstract_filter if it hasn't been consumed 

279 # yet and gen2keys includes tract. 

280 factory.addRule(PhysicalToAbstractFilterKeyHandler(self.filterDefinitions), 

281 instrument=self.getName(), gen2keys=("filter", "tract"), consume=("filter",)) 

282 return factory