Coverage for python/lsst/obs/lsst/

1# This file is part of obs_lsst.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <http://www.gnu.org/licenses/>.

22__all__ = ("LsstCam", "LsstCamImSim", "LsstCamPhoSim", "LsstTS8",

23 "Latiss", "LsstTS3", "LsstUCDCam", "LsstComCam", "LsstComCamSim",

24 "LsstCamSim")

26import datetime

27import hashlib

28import importlib.resources

30import lsst.obs.base.yamlCamera as yamlCamera

31from lsst.utils.introspection import get_full_type_name

32from lsst.obs.base import Instrument, VisitSystem

33from .filters import (LSSTCAM_FILTER_DEFINITIONS, LATISS_FILTER_DEFINITIONS,

34 LSSTCAM_IMSIM_FILTER_DEFINITIONS, TS3_FILTER_DEFINITIONS,

35 TS8_FILTER_DEFINITIONS, COMCAM_FILTER_DEFINITIONS,

36 GENERIC_FILTER_DEFINITIONS, UCD_FILTER_DEFINITIONS,

37 )

39from .translators import LatissTranslator, LsstCamTranslator, \

40 LsstUCDCamTranslator, LsstTS3Translator, LsstComCamTranslator, \

41 LsstCamPhoSimTranslator, LsstTS8Translator, LsstCamImSimTranslator, \

42 LsstComCamSimTranslator, LsstCamSimTranslator

44from .translators.lsst import GROUP_RE, TZERO_DATETIME

47class LsstCam(Instrument):

48 """Gen3 Butler specialization for the LSST Main Camera.

50 Parameters

51 ----------

52 camera : `lsst.cameraGeom.Camera`

53 Camera object from which to extract detector information.

54 filters : `list` of `FilterDefinition`

55 An ordered list of filters to define the set of PhysicalFilters

56 associated with this instrument in the registry.

58 While both the camera geometry and the set of filters associated with a

59 camera are expected to change with time in general, their Butler Registry

60 representations defined by an Instrument do not. Instead:

62 - We only extract names, IDs, and purposes from the detectors in the

63 camera, which should be static information that actually reflects

64 detector "slots" rather than the physical sensors themselves. Because

65 the distinction between physical sensors and slots is unimportant in

66 the vast majority of Butler use cases, we just use "detector" even

67 though the concept really maps better to "detector slot". Ideally in

68 the future this distinction between static and time-dependent

69 information would be encoded in cameraGeom itself (e.g. by making the

70 time-dependent Detector class inherit from a related class that only

71 carries static content).

73 - The Butler Registry is expected to contain physical_filter entries for

74 all filters an instrument has ever had, because we really only care

75 about which filters were used for particular observations, not which

76 filters were *available* at some point in the past. And changes in

77 individual filters over time will be captured as changes in their

78 TransmissionCurve datasets, not changes in the registry content (which

79 is really just a label). While at present Instrument and Registry

80 do not provide a way to add new physical_filters, they will in the

81 future.

82 """

83 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS

84 instrument = "LSSTCam"

85 policyName = "lsstCam"

86 translatorClass = LsstCamTranslator

87 obsDataPackage = "obs_lsst_data"

88 visitSystem = VisitSystem.BY_SEQ_START_END

90 @property

91 def configPaths(self):

92 return ["resource://lsst.obs.lsst/resources/config",

93 f"resource://lsst.obs.lsst/resources/config/{self.policyName}"]

95 @classmethod

96 def getName(cls):

97 # Docstring inherited from Instrument.getName

98 return cls.instrument

100 @classmethod

101 def getCamera(cls):

102 # Constructing a YAML camera takes a long time but we rely on

103 # yamlCamera to cache for us.

104 with importlib.resources.path(

105 "lsst.obs.lsst", f"resources/policy/{cls.policyName}.yaml"

106 ) as cameraYamlFile:

107 camera = yamlCamera.makeCamera(cameraYamlFile)

108 if camera.getName() != cls.getName():

109 raise RuntimeError(f"Expected to read camera geometry for {cls.instrument}"

110 f" but instead got geometry for {camera.getName()}")

111 return camera

112

113 def _make_default_dimension_packer(

114 self,

115 config_attr,

116 data_id,

117 is_exposure=None,

118 default="rubin",

119 ):

120 # Docstring inherited from Instrument._make_default_dimension_packer.

121 # Only difference is the change to default above.

122 return super()._make_default_dimension_packer(

123 config_attr,

124 data_id,

125 is_exposure=is_exposure,

126 default=default,

127 )

128

129 def getRawFormatter(self, dataId):

130 # Docstring inherited from Instrument.getRawFormatter

131 # local import to prevent circular dependency

132 from .rawFormatter import LsstCamRawFormatter

133 return LsstCamRawFormatter

134

135 def register(self, registry, update=False):

136 # Docstring inherited from Instrument.register

137 # The maximum values below make Gen3's ObservationDataIdPacker produce

138 # outputs that match Gen2's ccdExposureId.

139 obsMax = self.translatorClass.max_exposure_id()

140 # Make sure this is at least 1 to avoid non-uniqueness issues (e.g.

141 # for data ids that also get used in indexing).

142 detectorMax = max(self.translatorClass.DETECTOR_MAX, 1)

143 with registry.transaction():

144 registry.syncDimensionData(

145 "instrument",

146 {

147 "name": self.getName(),

148 "detector_max": detectorMax,

149 "visit_max": obsMax,

150 "exposure_max": obsMax,

151 "class_name": get_full_type_name(self),

152 "visit_system": None if self.visitSystem is None else self.visitSystem.value,

153 },

154 update=update

155 )

156 for detector in self.getCamera():

157 registry.syncDimensionData("detector", self.extractDetectorRecord(detector), update=update)

158

159 self._registerFilters(registry, update=update)

160

161 def extractDetectorRecord(self, camGeomDetector):

162 """Create a Gen3 Detector entry dict from a cameraGeom.Detector.

163 """

164 # All of the LSST instruments have detector names like R??_S??; we'll

165 # split them up here, and instruments with only one raft can override

166 # to change the group to something else if desired.

167 # Long-term, we should get these fields into cameraGeom separately

168 # so there's no need to specialize at this stage.

169 # They are separate in ObservationInfo

170 group, name = camGeomDetector.getName().split("_")

171

172 # getType() returns a pybind11-wrapped enum, which unfortunately

173 # has no way to extract the name of just the value (it's always

174 # prefixed by the enum type name).

175 purpose = str(camGeomDetector.getType()).split(".")[-1]

176

177 return dict(

178 instrument=self.getName(),

179 id=camGeomDetector.getId(),

180 full_name=camGeomDetector.getName(),

181 name_in_raft=name,

182 purpose=purpose,

183 raft=group,

184 )

185

186 @classmethod

187 def group_name_to_group_id(cls, group_name: str) -> int:

188 """Translate the exposure group name to an integer.

189

190 Parameters

191 ----------

192 group_name : `str`

193 The name of the exposure group.

194

195 Returns

196 -------

197 id : `int`

198 The exposure group name in integer form. This integer might be

199 used as an ID to uniquely identify the group in contexts where

200 a string can not be used.

201

202 Notes

203 -----

204 If given a group name that can be directly cast to an integer it

205 returns the integer. If the group name looks like an ISO date the ID

206 returned is seconds since an arbitrary recent epoch. Otherwise

207 the group name is hashed and the first 14 digits of the hash is

208 returned along with the length of the group name.

209 """

210 # If the group is an int we return it

211 try:

212 group_id = int(group_name)

213 return group_id

214 except ValueError:

215 pass

216

217 # A Group is defined as ISO date with an extension

218 # The integer must be the same for a given group so we can never

219 # use datetime_begin.

220 # Nominally a GROUPID looks like "ISODATE+N" where the +N is

221 # optional. This can be converted to seconds since epoch with

222 # N being zero-padded to 4 digits and appended (defaulting to 0).

223 # For early data lacking that form we hash the group and return

224 # the int.

225 matches_date = GROUP_RE.match(group_name)

226 if matches_date:

227 iso_str = matches_date.group(1)

228 fraction = matches_date.group(2)

229 n = matches_date.group(3)

230 if n is not None:

231 n = int(n)

232 else:

233 n = 0

234 iso = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%S")

235

236 tdelta = iso - TZERO_DATETIME

237 epoch = int(tdelta.total_seconds())

238

239 # Form the integer from EPOCH + 3 DIGIT FRAC + 0-pad N

240 group_id = int(f"{epoch}{fraction}{n:04d}")

241 else:

242 # Non-standard string so convert to numbers

243 # using a hash function. Use the first N hex digits

244 group_bytes = group_name.encode("us-ascii")

245 hasher = hashlib.blake2b(group_bytes)

246 # Need to be big enough it does not possibly clash with the

247 # date-based version above

248 digest = hasher.hexdigest()[:14]

249 group_id = int(digest, base=16)

250

251 # To help with hash collision, append the string length

252 group_id = int(f"{group_id}{len(group_name):02d}")

253

254 return group_id

255

256

257class LsstComCam(LsstCam):

258 """Gen3 Butler specialization for ComCam data.

259 """

260

261 filterDefinitions = COMCAM_FILTER_DEFINITIONS

262 instrument = "LSSTComCam"

263 policyName = "comCam"

264 translatorClass = LsstComCamTranslator

265

266 def getRawFormatter(self, dataId):

267 # local import to prevent circular dependency

268 from .rawFormatter import LsstComCamRawFormatter

269 return LsstComCamRawFormatter

270

271

272class LsstComCamSim(LsstCam):

273 """Gen3 Butler specialization for ComCamSim data.

274 """

275

276 filterDefinitions = COMCAM_FILTER_DEFINITIONS

277 instrument = "LSSTComCamSim"

278 policyName = "comCamSim"

279 translatorClass = LsstComCamSimTranslator

280

281 def getRawFormatter(self, dataId):

282 # local import to prevent circular dependency

283 from .rawFormatter import LsstComCamSimRawFormatter

284 return LsstComCamSimRawFormatter

285

286

287class LsstCamSim(LsstCam):

288 """Gen3 Butler specialization for LSSTCamSim data.

289 """

290

291 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS

292 instrument = "LSSTCamSim"

293 policyName = "lsstCamSim"

294 translatorClass = LsstCamSimTranslator

295

296 def getRawFormatter(self, dataId):

297 # local import to prevent circular dependency

298 from .rawFormatter import LsstCamSimRawFormatter

299 return LsstCamSimRawFormatter

300

301

302class LsstCamImSim(LsstCam):

303 """Gen3 Butler specialization for ImSim simulations.

304 """

305

306 instrument = "LSSTCam-imSim"

307 policyName = "imsim"

308 translatorClass = LsstCamImSimTranslator

309 filterDefinitions = LSSTCAM_IMSIM_FILTER_DEFINITIONS

310 visitSystem = VisitSystem.ONE_TO_ONE

311

312 def getRawFormatter(self, dataId):

313 # local import to prevent circular dependency

314 from .rawFormatter import LsstCamImSimRawFormatter

315 return LsstCamImSimRawFormatter

316

317 def _make_default_dimension_packer(

318 self,

319 config_attr,

320 data_id,

321 is_exposure=None,

322 default="observation",

323 ):

324 # Docstring inherited from Instrument._make_default_dimension_packer.

325 # Only difference is the change to default above (which reverts back

326 # the default in lsst.pipe.base.Instrument).

327 return super()._make_default_dimension_packer(

328 config_attr,

329 data_id,

330 is_exposure=is_exposure,

331 default=default,

332 )

333

334

335class LsstCamPhoSim(LsstCam):

336 """Gen3 Butler specialization for Phosim simulations.

337 """

338

339 instrument = "LSSTCam-PhoSim"

340 policyName = "phosim"

341 translatorClass = LsstCamPhoSimTranslator

342 filterDefinitions = GENERIC_FILTER_DEFINITIONS

343 visitSystem = VisitSystem.ONE_TO_ONE

344

345 def getRawFormatter(self, dataId):

346 # local import to prevent circular dependency

347 from .rawFormatter import LsstCamPhoSimRawFormatter

348 return LsstCamPhoSimRawFormatter

349

350 def _make_default_dimension_packer(

351 self,

352 config_attr,

353 data_id,

354 is_exposure=None,

355 default="observation",

356 ):

357 # Docstring inherited from Instrument._make_default_dimension_packer.

358 # Only difference is the change to default above (which reverts back

359 # the default in lsst.pipe.base.Instrument).

360 return super()._make_default_dimension_packer(

361 config_attr,

362 data_id,

363 is_exposure=is_exposure,

364 default=default,

365 )

366

367

368class LsstTS8(LsstCam):

369 """Gen3 Butler specialization for raft test stand data.

370 """

371

372 filterDefinitions = TS8_FILTER_DEFINITIONS

373 instrument = "LSST-TS8"

374 policyName = "ts8"

375 translatorClass = LsstTS8Translator

376 visitSystem = VisitSystem.ONE_TO_ONE

377

378 def getRawFormatter(self, dataId):

379 # local import to prevent circular dependency

380 from .rawFormatter import LsstTS8RawFormatter

381 return LsstTS8RawFormatter

382

383 def _make_default_dimension_packer(

384 self,

385 config_attr,

386 data_id,

387 is_exposure=None,

388 default="observation",

389 ):

390 # Docstring inherited from Instrument._make_default_dimension_packer.

391 # Only difference is the change to default above (which reverts back

392 # the default in lsst.pipe.base.Instrument).

393 return super()._make_default_dimension_packer(

394 config_attr,

395 data_id,

396 is_exposure=is_exposure,

397 default=default,

398 )

399

400

401class LsstUCDCam(LsstCam):

402 """Gen3 Butler specialization for UCDCam test stand data.

403 """

404 filterDefinitions = UCD_FILTER_DEFINITIONS

405 instrument = "LSST-UCDCam"

406 policyName = "ucd"

407 translatorClass = LsstUCDCamTranslator

408 visitSystem = VisitSystem.ONE_TO_ONE

409

410 def getRawFormatter(self, dataId):

411 # local import to prevent circular dependency

412 from .rawFormatter import LsstUCDCamRawFormatter

413 return LsstUCDCamRawFormatter

414

415 def _make_default_dimension_packer(

416 self,

417 config_attr,

418 data_id,

419 is_exposure=None,

420 default="observation",

421 ):

422 # Docstring inherited from Instrument._make_default_dimension_packer.

423 # Only difference is the change to default above (which reverts back

424 # the default in lsst.pipe.base.Instrument).

425 return super()._make_default_dimension_packer(

426 config_attr,

427 data_id,

428 is_exposure=is_exposure,

429 default=default,

430 )

431

432

433class LsstTS3(LsstCam):

434 """Gen3 Butler specialization for TS3 test stand data.

435 """

436

437 filterDefinitions = TS3_FILTER_DEFINITIONS

438 instrument = "LSST-TS3"

439 policyName = "ts3"

440 translatorClass = LsstTS3Translator

441 visitSystem = VisitSystem.ONE_TO_ONE

442

443 def getRawFormatter(self, dataId):

444 # local import to prevent circular dependency

445 from .rawFormatter import LsstTS3RawFormatter

446 return LsstTS3RawFormatter

447

448 def _make_default_dimension_packer(

449 self,

450 config_attr,

451 data_id,

452 is_exposure=None,

453 default="observation",

454 ):

455 # Docstring inherited from Instrument._make_default_dimension_packer.

456 # Only difference is the change to default above (which reverts back

457 # the default in lsst.pipe.base.Instrument).

458 return super()._make_default_dimension_packer(

459 config_attr,

460 data_id,

461 is_exposure=is_exposure,

462 default=default,

463 )

464

465

466class Latiss(LsstCam):

467 """Gen3 Butler specialization for AuxTel LATISS data.

468 """

469 filterDefinitions = LATISS_FILTER_DEFINITIONS

470 instrument = "LATISS"

471 policyName = "latiss"

472 translatorClass = LatissTranslator

473

474 def extractDetectorRecord(self, camGeomDetector):

475 # Override to remove group (raft) name, because LATISS only has one

476 # detector.

477 record = super().extractDetectorRecord(camGeomDetector)

478 record["raft"] = None

479 record["name_in_raft"] = record["full_name"]

480 return record

481

482 def getRawFormatter(self, dataId):

483 # local import to prevent circular dependency

484 from .rawFormatter import LatissRawFormatter

485 return LatissRawFormatter

Coverage for python / lsst / obs / lsst / _instrument.py: 55%

164 statements