Coverage for tests/test_ingest.py: 53%
157 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-25 01:21 -0800
1# This file is part of obs_lsst.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for LSST raw data ingest.
23"""
25import unittest
26import os
27import tempfile
28import lsst.utils.tests
30from lsst.afw.math import flipImage
31from lsst.afw.cameraGeom import AmplifierGeometryComparison
32from lsst.daf.butler import Butler, DataCoordinate
33from lsst.daf.butler.cli.butler import cli as butlerCli
34from lsst.daf.butler.cli.utils import LogCliRunner
35from lsst.obs.base.ingest_tests import IngestTestBase
36from lsst.ip.isr import PhotodiodeCalib, ShutterMotionProfile
37import lsst.afw.cameraGeom.testUtils # for injected test asserts
38import lsst.obs.lsst
40TESTDIR = os.path.abspath(os.path.dirname(__file__))
41DATAROOT = os.path.join(TESTDIR, os.path.pardir, "data", "input")
44class LatissIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase):
46 curatedCalibrationDatasetTypes = ("camera", "defects")
47 instrumentClassName = "lsst.obs.lsst.Latiss"
48 ingestDir = TESTDIR
49 file = os.path.join(DATAROOT, "latiss", "raw", "2018-09-20", "3018092000065-det000.fits")
50 dataIds = [dict(instrument="LATISS", exposure=3018092000065, detector=0)]
51 filterLabel = lsst.afw.image.FilterLabel(band="unknown", physical="unknown~unknown")
53 def checkRepo(self, files=None):
54 # Test amp parameter implementation for the LSST raw formatter. This
55 # is the same for all instruments, so repeating it in other test cases
56 # is wasteful.
57 butler = Butler(self.root, run=self.outputRun)
58 ref = butler.find_dataset("raw", self.dataIds[0])
59 full_assembled = butler.get(ref)
60 unassembled_detector = self.instrumentClass().getCamera()[ref.dataId["detector"]]
61 assembled_detector = full_assembled.getDetector()
62 for unassembled_amp, assembled_amp in zip(unassembled_detector, assembled_detector):
63 # Check that we're testing what we think we're testing: these
64 # amps should differ in assembly state (offsets, flips), and they
65 # _may_ differ in fundamental geometry if we had to patch the
66 # overscan region sizes.
67 comparison = unassembled_amp.compareGeometry(assembled_amp)
68 self.assertTrue(comparison & AmplifierGeometryComparison.ASSEMBLY_DIFFERS)
69 assembled_subimage = butler.get(ref, parameters={"amp": assembled_amp})
70 unassembled_subimage = butler.get(ref, parameters={"amp": unassembled_amp.getName()})
71 self.assertEqual(len(assembled_subimage.getDetector()), 1)
72 self.assertEqual(len(unassembled_subimage.getDetector()), 1)
73 self.assertEqual(len(assembled_subimage.getDetector()), 1)
74 self.assertEqual(len(unassembled_subimage.getDetector()), 1)
75 self.assertImagesEqual(assembled_subimage.image, full_assembled.image[assembled_amp.getRawBBox()])
76 self.assertImagesEqual(
77 unassembled_subimage.image,
78 flipImage(
79 full_assembled.image[assembled_amp.getRawBBox()],
80 flipLR=unassembled_amp.getRawFlipX(),
81 flipTB=unassembled_amp.getRawFlipY(),
82 ),
83 )
84 self.assertAmplifiersEqual(assembled_subimage.getDetector()[0], assembled_amp)
85 if comparison & comparison.REGIONS_DIFFER:
86 # We needed to patch overscans, but unassembled_amp (which
87 # comes straight from the camera) won't have those patches, so
88 # we can't compare it to the amp attached to
89 # unassembled_subimage (which does have those patches).
90 comparison2 = unassembled_subimage.getDetector()[0].compareGeometry(unassembled_amp)
92 self.assertTrue(comparison2 & AmplifierGeometryComparison.REGIONS_DIFFER)
93 # ...and that unassembled_subimage's amp has the same regions
94 # (after accounting for assembly/orientation) as assembled_amp.
95 comparison3 = unassembled_subimage.getDetector()[0].compareGeometry(assembled_amp)
96 self.assertTrue(comparison3 & AmplifierGeometryComparison.ASSEMBLY_DIFFERS)
97 self.assertFalse(comparison3 & AmplifierGeometryComparison.REGIONS_DIFFER)
98 else:
99 self.assertAmplifiersEqual(unassembled_subimage.getDetector()[0], unassembled_amp)
102class Ts3IngestTestCase(IngestTestBase, lsst.utils.tests.TestCase):
104 curatedCalibrationDatasetTypes = ("camera",)
105 instrumentClassName = "lsst.obs.lsst.LsstTS3"
106 ingestDir = TESTDIR
107 file = os.path.join(DATAROOT, "ts3", "raw", "2018-11-15", "201811151255111-R433-S00-det433.fits")
108 dataIds = [dict(instrument="LSST-TS3", exposure=201811151255111, detector=433)]
109 filterLabel = lsst.afw.image.FilterLabel(physical="550CutOn")
112class ComCamIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase):
114 curatedCalibrationDatasetTypes = ("camera",)
115 instrumentClassName = "lsst.obs.lsst.LsstComCam"
116 ingestDir = TESTDIR
117 file = os.path.join(DATAROOT, "comCam", "raw", "2019-05-30",
118 "3019053000001", "3019053000001-R22-S00-det000.fits")
119 dataIds = [dict(instrument="LSSTComCam", exposure=3019053000001, detector=0)]
120 filterLabel = lsst.afw.image.FilterLabel(physical="unknown", band="unknown")
123class ComCamSimIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase):
125 curatedCalibrationDatasetTypes = ("camera",)
126 instrumentClassName = "lsst.obs.lsst.LsstComCamSim"
127 ingestDir = TESTDIR
128 file = os.path.join(DATAROOT, "comCamSim", "raw", "2024-04-04",
129 "7024040400780", "CC_S_20240404_000780_R22_S01.fits")
130 dataIds = [dict(instrument="LSSTComCamSim", exposure=7024040400780, detector=1)]
131 filterLabel = lsst.afw.image.FilterLabel(physical="r_03", band="r")
133 @property
134 def visits(self):
135 butler = Butler(self.root, collections=[self.outputRun])
136 return {
137 DataCoordinate.standardize(
138 instrument="LSSTComCamSim",
139 visit=7024040400780,
140 universe=butler.dimensions
141 ): [
142 DataCoordinate.standardize(
143 instrument="LSSTComCamSim",
144 exposure=7024040400780,
145 universe=butler.dimensions
146 )
147 ]
148 }
151class LSSTCamIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase):
153 curatedCalibrationDatasetTypes = ("camera",)
154 instrumentClassName = "lsst.obs.lsst.LsstCam"
155 ingestDir = TESTDIR
156 file = os.path.join(DATAROOT, "lsstCam", "raw", "2019-03-19",
157 "3019031900001", "3019031900001-R10-S02-det029.fits")
158 dataIds = [dict(instrument="LSSTCam", exposure=3019031900001, detector=29)]
159 filterLabel = lsst.afw.image.FilterLabel(physical="unknown", band="unknown")
162class LSSTCamSimIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase):
164 curatedCalibrationDatasetTypes = ("camera",)
165 instrumentClassName = "lsst.obs.lsst.LsstCamSim"
166 ingestDir = TESTDIR
167 file = os.path.join(DATAROOT, "lsstCamSim", "raw", "2024-03-21",
168 "7024032100720", "7024032100720-R22-S11-det094.fits.fz")
169 dataIds = [dict(instrument="LSSTCamSim", exposure=7024032100720, detector=94)]
170 filterLabel = lsst.afw.image.FilterLabel(physical="r_57", band="r")
172 @property
173 def visits(self):
174 butler = Butler(self.root, collections=[self.outputRun])
175 return {
176 DataCoordinate.standardize(
177 instrument="LSSTCamSim",
178 visit=7024032100720,
179 universe=butler.dimensions
180 ): [
181 DataCoordinate.standardize(
182 instrument="LSSTCamSim",
183 exposure=7024032100720,
184 universe=butler.dimensions
185 )
186 ]
187 }
190class LSSTCamPhotodiodeIngestTestCase(lsst.utils.tests.TestCase):
191 instrumentClassName = "lsst.obs.lsst.LsstCam"
192 rawIngestTask = "lsst.obs.base.RawIngestTask"
193 ingestDir = TESTDIR
194 file = os.path.join(DATAROOT, "lsstCam", "raw", "2021-12-12",
195 "30211212000310", "30211212000310-R22-S22-det098.fits")
196 dataIds = [dict(instrument="LSSTCam", exposure=3021121200310, detector=98)]
197 filterLabel = lsst.afw.image.FilterLabel(physical="SDSSi", band="i")
198 pdPath = os.path.join(DATAROOT, "lsstCam", "raw")
200 def setUp(self):
201 """Setup for lightweight photodiode ingest task.
203 This will create the repo and register the instrument.
204 """
205 self.root = tempfile.mkdtemp(dir=self.ingestDir)
207 # Create Repo
208 runner = LogCliRunner()
209 result = runner.invoke(butlerCli, ["create", self.root])
210 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
212 # Register Instrument
213 runner = LogCliRunner()
214 result = runner.invoke(butlerCli, ["register-instrument", self.root, self.instrumentClassName])
215 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
217 def testPhotodiodeFailure(self):
218 """Test ingest to a repo missing exposure information will raise.
219 """
220 runner = LogCliRunner()
221 result = runner.invoke(
222 butlerCli,
223 [
224 "ingest-photodiode",
225 self.root,
226 self.instrumentClassName,
227 self.pdPath,
228 ],
229 )
230 self.assertEqual(result.exit_code, 1, f"output: {result.output} exception: {result.exception}")
232 def testPhotodiode(self):
233 """Test ingest to a repo with the exposure information will not raise.
234 """
235 # Ingest raw to provide exposure information.
236 outputRun = "raw_ingest_" + self.id()
237 runner = LogCliRunner()
238 result = runner.invoke(
239 butlerCli,
240 [
241 "ingest-raws",
242 self.root,
243 self.file,
244 "--output-run",
245 outputRun,
246 "--ingest-task",
247 self.rawIngestTask,
248 ],
249 )
250 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
252 # Ingest photodiode matching this exposure.
253 runner = LogCliRunner()
254 result = runner.invoke(
255 butlerCli,
256 [
257 "ingest-photodiode",
258 self.root,
259 self.instrumentClassName,
260 self.pdPath,
261 ],
262 )
263 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
265 # Confirm that we can retrieve the ingested photodiode, and
266 # that it has the correct type.
267 butler = Butler(self.root, run="LSSTCam/calib/photodiode")
268 getResult = butler.get('photodiode', dataId=self.dataIds[0])
269 self.assertIsInstance(getResult, PhotodiodeCalib)
272class LSSTCamShutterMotionIngestTestCase(lsst.utils.tests.TestCase):
273 instrumentClassName = "lsst.obs.lsst.LsstCam"
274 rawIngestTask = "lsst.obs.base.RawIngestTask"
275 ingestDir = TESTDIR
276 file = os.path.join(DATAROOT, "lsstCam", "raw", "2021-12-12",
277 "30211212000310", "30211212000310-R22-S22-det098.fits")
278 dataIds = [dict(instrument="LSSTCam", exposure=3021121200310, detector=98)]
279 filterLabel = lsst.afw.image.FilterLabel(physical="SDSSi", band="i")
280 smpPath = os.path.join(DATAROOT, "lsstCam", "raw")
282 def setUp(self):
283 """Setup for lightweight shutter motion ingest task.
285 This will create the repo and register the instrument.
286 """
287 self.root = tempfile.mkdtemp(dir=self.ingestDir)
289 # Create Repo
290 runner = LogCliRunner()
291 result = runner.invoke(butlerCli, ["create", self.root])
292 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
294 # Register Instrument
295 runner = LogCliRunner()
296 result = runner.invoke(butlerCli, ["register-instrument", self.root, self.instrumentClassName])
297 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
299 def testShutterMotionFailure(self):
300 """Test ingest to a repo missing exposure information will raise.
301 """
302 runner = LogCliRunner()
303 result = runner.invoke(
304 butlerCli,
305 [
306 "ingest-shuttermotion",
307 self.root,
308 self.instrumentClassName,
309 self.smpPath,
310 "-c",
311 "doRaiseOnMissingExposure=True",
312 ],
313 )
314 self.assertEqual(result.exit_code, 1, f"output: {result.output} exception: {result.exception}")
316 def testShutterMotion(self):
317 """Test ingest to a repo with the exposure information will not raise.
318 """
319 # Ingest raw to provide exposure information.
320 outputRun = "raw_ingest_" + self.id()
321 runner = LogCliRunner()
322 result = runner.invoke(
323 butlerCli,
324 [
325 "ingest-raws",
326 self.root,
327 self.file,
328 "--output-run",
329 outputRun,
330 "--ingest-task",
331 self.rawIngestTask,
332 ],
333 )
334 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
336 # Ingest photodiode matching this exposure.
337 runner = LogCliRunner()
338 result = runner.invoke(
339 butlerCli,
340 [
341 "ingest-shuttermotion",
342 self.root,
343 self.instrumentClassName,
344 self.smpPath,
345 "-c",
346 "doRaiseOnMissingExposure=True",
347 ],
348 )
349 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
351 # Confirm that we can retrieve the ingested photodiode, and
352 # that it has the correct type.
353 butler = Butler(self.root, run="LSSTCam/calib/shutterMotion")
354 getResult = butler.get('shutterMotionProfileOpen', dataId=self.dataIds[0])
355 self.assertIsInstance(getResult, ShutterMotionProfile)
358def setup_module(module):
359 lsst.utils.tests.init()
362if __name__ == "__main__": 362 ↛ 363line 362 didn't jump to line 363 because the condition on line 362 was never true
363 lsst.utils.tests.init()
364 unittest.main()