Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_subaru. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Gen3 Butler registry declarations for Hyper Suprime-Cam. 

23""" 

24 

25__all__ = ("HyperSuprimeCam",) 

26 

27import os 

28import pickle 

29import logging 

30 

31from functools import lru_cache 

32 

33import astropy.time 

34from lsst.utils import getPackageDir 

35from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig 

36from lsst.daf.butler import (DatasetType, DataCoordinate, FileDataset, DatasetRef, 

37 CollectionType, Timespan) 

38from lsst.daf.butler.core.utils import getFullTypeName 

39from lsst.obs.base import Instrument 

40from lsst.obs.base.gen2to3 import TranslatorFactory, PhysicalFilterToBandKeyHandler 

41 

42from ..hsc.hscPupil import HscPupilFactory 

43from ..hsc.hscFilters import HSC_FILTER_DEFINITIONS 

44from ..hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission, 

45 getFilterTransmission, getAtmosphereTransmission) 

46from .strayLight.formatter import SubaruStrayLightDataFormatter 

47 

48log = logging.getLogger(__name__) 

49 

50 

51class HyperSuprimeCam(Instrument): 

52 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam. 

53 """ 

54 

55 policyName = "hsc" 

56 obsDataPackage = "obs_subaru_data" 

57 filterDefinitions = HSC_FILTER_DEFINITIONS 

58 additionalCuratedDatasetTypes = ("bfKernel", "transmission_optics", "transmission_sensor", 

59 "transmission_filter", "transmission_atmosphere", "yBackground") 

60 

61 def __init__(self, **kwargs): 

62 super().__init__(**kwargs) 

63 packageDir = getPackageDir("obs_subaru") 

64 self.configPaths = [os.path.join(packageDir, "config"), 

65 os.path.join(packageDir, "config", self.policyName)] 

66 

67 @classmethod 

68 def getName(cls): 

69 # Docstring inherited from Instrument.getName 

70 return "HSC" 

71 

72 def register(self, registry): 

73 # Docstring inherited from Instrument.register 

74 camera = self.getCamera() 

75 # The maximum values below make Gen3's ObservationDataIdPacker produce 

76 # outputs that match Gen2's ccdExposureId. 

77 obsMax = 21474800 

78 registry.insertDimensionData( 

79 "instrument", 

80 { 

81 "name": self.getName(), 

82 "detector_max": 200, 

83 "visit_max": obsMax, 

84 "exposure_max": obsMax, 

85 "class_name": getFullTypeName(self), 

86 } 

87 ) 

88 registry.insertDimensionData( 

89 "detector", 

90 *[ 

91 { 

92 "instrument": self.getName(), 

93 "id": detector.getId(), 

94 "full_name": detector.getName(), 

95 # TODO: make sure these definitions are consistent with those 

96 # extracted by astro_metadata_translator, and test that they 

97 # remain consistent somehow. 

98 "name_in_raft": detector.getName().split("_")[1], 

99 "raft": detector.getName().split("_")[0], 

100 "purpose": str(detector.getType()).split(".")[-1], 

101 } 

102 for detector in camera 

103 ] 

104 ) 

105 self._registerFilters(registry) 

106 

107 def getRawFormatter(self, dataId): 

108 # Docstring inherited from Instrument.getRawFormatter 

109 # Import the formatter here to prevent a circular dependency. 

110 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter 

111 if dataId["detector"] in (100, 101, 102, 103): 

112 return HyperSuprimeCamCornerRawFormatter 

113 else: 

114 return HyperSuprimeCamRawFormatter 

115 

116 def getCamera(self): 

117 """Retrieve the cameraGeom representation of HSC. 

118 

119 This is a temporary API that should go away once obs_ packages have 

120 a standardized approach to writing versioned cameras to a Gen3 repo. 

121 """ 

122 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "camera") 

123 return self._getCameraFromPath(path) 

124 

125 @staticmethod 

126 @lru_cache() 

127 def _getCameraFromPath(path): 

128 """Return the camera geometry given solely the path to the location 

129 of that definition.""" 

130 config = CameraConfig() 

131 config.load(os.path.join(path, "camera.py")) 

132 return makeCameraFromPath( 

133 cameraConfig=config, 

134 ampInfoPath=path, 

135 shortNameFunc=lambda name: name.replace(" ", "_"), 

136 pupilFactoryClass=HscPupilFactory 

137 ) 

138 

139 def getBrighterFatterKernel(self): 

140 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`. 

141 

142 This is a temporary API that should go away once obs_ packages have 

143 a standardized approach to writing versioned kernels to a Gen3 repo. 

144 """ 

145 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "brighter_fatter_kernel.pkl") 

146 with open(path, "rb") as fd: 

147 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2 

148 return kernel 

149 

150 def writeAdditionalCuratedCalibrations(self, butler, collection=None, suffixes=()): 

151 # Register the CALIBRATION collection that adds validity ranges. 

152 # This does nothing if it is already registered. 

153 if collection is None: 

154 collection = self.makeCalibrationCollectionName(*suffixes) 

155 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION) 

156 

157 # Register the RUN collection that holds these datasets directly. We 

158 # only need one because all of these datasets have the same (unbounded) 

159 # validity range right now. 

160 run = self.makeUnboundedCalibrationRunName(*suffixes) 

161 butler.registry.registerRun(run) 

162 baseDataId = butler.registry.expandDataId(instrument=self.getName()) 

163 refs = [] 

164 

165 # Write brighter-fatter kernel, with an infinite validity range. 

166 datasetType = DatasetType("bfKernel", ("instrument",), "NumpyArray", 

167 universe=butler.registry.dimensions, 

168 isCalibration=True) 

169 butler.registry.registerDatasetType(datasetType) 

170 

171 # Load and then put instead of just moving the file in part to ensure 

172 # the version in-repo is written with Python 3 and does not need 

173 # `encoding='latin1'` to be read. 

174 bfKernel = self.getBrighterFatterKernel() 

175 refs.append(butler.put(bfKernel, datasetType, baseDataId, run=run)) 

176 

177 # The following iterate over the values of the dictionaries returned by the transmission functions 

178 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates, 

179 # which do not give the proper notion of validity. As such unbounded calibration labels are used 

180 # when inserting into the database. In the future these could and probably should be updated to 

181 # properly account for what ranges are considered valid. 

182 

183 # Write optical transmissions 

184 opticsTransmissions = getOpticsTransmission() 

185 datasetType = DatasetType("transmission_optics", 

186 ("instrument",), 

187 "TransmissionCurve", 

188 universe=butler.registry.dimensions, 

189 isCalibration=True) 

190 butler.registry.registerDatasetType(datasetType) 

191 for entry in opticsTransmissions.values(): 

192 if entry is None: 

193 continue 

194 refs.append(butler.put(entry, datasetType, baseDataId, run=run)) 

195 

196 # Write transmission sensor 

197 sensorTransmissions = getSensorTransmission() 

198 datasetType = DatasetType("transmission_sensor", 

199 ("instrument", "detector",), 

200 "TransmissionCurve", 

201 universe=butler.registry.dimensions, 

202 isCalibration=True) 

203 butler.registry.registerDatasetType(datasetType) 

204 for entry in sensorTransmissions.values(): 

205 if entry is None: 

206 continue 

207 for sensor, curve in entry.items(): 

208 dataId = DataCoordinate.standardize(baseDataId, detector=sensor) 

209 refs.append(butler.put(curve, datasetType, dataId, run=run)) 

210 

211 # Write filter transmissions 

212 filterTransmissions = getFilterTransmission() 

213 datasetType = DatasetType("transmission_filter", 

214 ("instrument", "physical_filter",), 

215 "TransmissionCurve", 

216 universe=butler.registry.dimensions, 

217 isCalibration=True) 

218 butler.registry.registerDatasetType(datasetType) 

219 for entry in filterTransmissions.values(): 

220 if entry is None: 

221 continue 

222 for band, curve in entry.items(): 

223 dataId = DataCoordinate.standardize(baseDataId, physical_filter=band) 

224 refs.append(butler.put(curve, datasetType, dataId, run=run)) 

225 

226 # Write atmospheric transmissions 

227 atmosphericTransmissions = getAtmosphereTransmission() 

228 datasetType = DatasetType("transmission_atmosphere", ("instrument",), 

229 "TransmissionCurve", 

230 universe=butler.registry.dimensions, 

231 isCalibration=True) 

232 butler.registry.registerDatasetType(datasetType) 

233 for entry in atmosphericTransmissions.values(): 

234 if entry is None: 

235 continue 

236 refs.append(butler.put(entry, datasetType, {"instrument": self.getName()}, run=run)) 

237 

238 # Associate all datasets with the unbounded validity range. 

239 butler.registry.certify(collection, refs, Timespan(begin=None, end=None)) 

240 

241 def ingestStrayLightData(self, butler, directory, *, transfer=None, collection=None, suffixes=()): 

242 """Ingest externally-produced y-band stray light data files into 

243 a data repository. 

244 

245 Parameters 

246 ---------- 

247 butler : `lsst.daf.butler.Butler` 

248 Butler initialized with the collection to ingest into. 

249 directory : `str` 

250 Directory containing yBackground-*.fits files. 

251 transfer : `str`, optional 

252 If not `None`, must be one of 'move', 'copy', 'hardlink', or 

253 'symlink', indicating how to transfer the files. 

254 collection : `str`, optional 

255 Name to use for the calibration collection that associates all 

256 datasets with a validity range. If this collection already exists, 

257 it must be a `~CollectionType.CALIBRATION` collection, and it must 

258 not have any datasets that would conflict with those inserted by 

259 this method. If `None`, a collection name is worked out 

260 automatically from the instrument name and other metadata by 

261 calling ``makeCuratedCalibrationCollectionName``, but this 

262 default name may not work well for long-lived repositories unless 

263 one or more ``suffixes`` are also provided (and changed every time 

264 curated calibrations are ingested). 

265 suffixes : `Sequence` [ `str` ], optional 

266 Name suffixes to append to collection names, after concatenating 

267 them with the standard collection name delimeter. If provided, 

268 these are appended to the names of the `~CollectionType.RUN` 

269 collections that datasets are inserted directly into, as well the 

270 `~CollectionType.CALIBRATION` collection if it is generated 

271 automatically (i.e. if ``collection is None``). 

272 """ 

273 # Register the CALIBRATION collection that adds validity ranges. 

274 # This does nothing if it is already registered. 

275 if collection is None: 

276 collection = self.makeCalibrationCollectionName(*suffixes) 

277 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION) 

278 

279 # Register the RUN collection that holds these datasets directly. We 

280 # only need one because there is only one validity range and hence no 

281 # data ID conflicts even when there are no validity ranges. 

282 run = self.makeUnboundedCalibrationRunName(*suffixes) 

283 butler.registry.registerRun(run) 

284 

285 # LEDs covered up around 2018-01-01, no need for correctin after that 

286 # date. 

287 timespan = Timespan(begin=None, end=astropy.time.Time("2018-01-01", format="iso", scale="tai")) 

288 datasets = [] 

289 # TODO: should we use a more generic name for the dataset type? 

290 # This is just the (rather HSC-specific) name used in Gen2, and while 

291 # the instances of this dataset are camera-specific, the datasetType 

292 # (which is used in the generic IsrTask) should not be. 

293 datasetType = DatasetType("yBackground", 

294 dimensions=("physical_filter", "detector",), 

295 storageClass="StrayLightData", 

296 universe=butler.registry.dimensions, 

297 isCalibration=True) 

298 for detector in self.getCamera(): 

299 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits") 

300 if not os.path.exists(path): 

301 log.warning(f"No stray light data found for detector {detector.getId()} @ {path}.") 

302 continue 

303 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(), 

304 "detector": detector.getId(), 

305 "physical_filter": "HSC-Y"}) 

306 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter)) 

307 butler.registry.registerDatasetType(datasetType) 

308 with butler.transaction(): 

309 butler.ingest(*datasets, transfer=transfer, run=run) 

310 refs = [] 

311 for dataset in datasets: 

312 refs.extend(dataset.refs) 

313 butler.registry.certify(collection, refs, timespan) 

314 

315 def makeDataIdTranslatorFactory(self) -> TranslatorFactory: 

316 # Docstring inherited from lsst.obs.base.Instrument. 

317 factory = TranslatorFactory() 

318 factory.addGenericInstrumentRules(self.getName()) 

319 # Translate Gen2 `filter` to band if it hasn't been consumed 

320 # yet and gen2keys includes tract. 

321 factory.addRule(PhysicalFilterToBandKeyHandler(self.filterDefinitions), 

322 instrument=self.getName(), gen2keys=("filter", "tract"), consume=("filter",)) 

323 return factory