Coverage for tests/test_ingest.py: 53%
105 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-09 02:58 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-09 02:58 -0800
1# This file is part of obs_lsst.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for LSST raw data ingest.
23"""
25import unittest
26import os
27import tempfile
28import lsst.utils.tests
30from lsst.afw.math import flipImage
31from lsst.afw.cameraGeom import AmplifierGeometryComparison
32from lsst.daf.butler import Butler
33from lsst.daf.butler.cli.butler import cli as butlerCli
34from lsst.daf.butler.cli.utils import LogCliRunner
35from lsst.obs.base.ingest_tests import IngestTestBase
36from lsst.ip.isr import PhotodiodeCalib
37import lsst.afw.cameraGeom.testUtils # for injected test asserts
38import lsst.obs.lsst
40TESTDIR = os.path.abspath(os.path.dirname(__file__))
41DATAROOT = os.path.join(TESTDIR, os.path.pardir, "data", "input")
44class LatissIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase):
46 curatedCalibrationDatasetTypes = ("camera", "defects")
47 instrumentClassName = "lsst.obs.lsst.Latiss"
48 ingestDir = TESTDIR
49 file = os.path.join(DATAROOT, "latiss", "raw", "2018-09-20", "3018092000065-det000.fits")
50 dataIds = [dict(instrument="LATISS", exposure=3018092000065, detector=0)]
51 filterLabel = lsst.afw.image.FilterLabel(band="unknown", physical="unknown~unknown")
53 def checkRepo(self, files=None):
54 # Test amp parameter implementation for the LSST raw formatter. This
55 # is the same for all instruments, so repeating it in other test cases
56 # is wasteful.
57 butler = Butler(self.root, run=self.outputRun)
58 ref = butler.registry.findDataset("raw", self.dataIds[0])
59 full_assembled = butler.getDirect(ref)
60 unassembled_detector = self.instrumentClass().getCamera()[ref.dataId["detector"]]
61 assembled_detector = full_assembled.getDetector()
62 for unassembled_amp, assembled_amp in zip(unassembled_detector, assembled_detector):
63 # Check that we're testing what we think we're testing: these
64 # amps should differ in assembly state (offsets, flips), and they
65 # _may_ differ in fundamental geometry if we had to patch the
66 # overscan region sizes.
67 comparison = unassembled_amp.compareGeometry(assembled_amp)
68 self.assertTrue(comparison & AmplifierGeometryComparison.ASSEMBLY_DIFFERS)
69 assembled_subimage = butler.getDirect(ref, parameters={"amp": assembled_amp})
70 unassembled_subimage = butler.getDirect(ref, parameters={"amp": unassembled_amp.getName()})
71 self.assertEqual(len(assembled_subimage.getDetector()), 1)
72 self.assertEqual(len(unassembled_subimage.getDetector()), 1)
73 self.assertEqual(len(assembled_subimage.getDetector()), 1)
74 self.assertEqual(len(unassembled_subimage.getDetector()), 1)
75 self.assertImagesEqual(assembled_subimage.image, full_assembled.image[assembled_amp.getRawBBox()])
76 self.assertImagesEqual(
77 unassembled_subimage.image,
78 flipImage(
79 full_assembled.image[assembled_amp.getRawBBox()],
80 flipLR=unassembled_amp.getRawFlipX(),
81 flipTB=unassembled_amp.getRawFlipY(),
82 ),
83 )
84 self.assertAmplifiersEqual(assembled_subimage.getDetector()[0], assembled_amp)
85 if comparison & comparison.REGIONS_DIFFER:
86 # We needed to patch overscans, but unassembled_amp (which
87 # comes straight from the camera) won't have those patches, so
88 # we can't compare it to the amp attached to
89 # unassembled_subimage (which does have those patches).
90 comparison2 = unassembled_subimage.getDetector()[0].compareGeometry(unassembled_amp)
92 self.assertTrue(comparison2 & AmplifierGeometryComparison.REGIONS_DIFFER)
93 # ...and that unassembled_subimage's amp has the same regions
94 # (after accounting for assembly/orientation) as assembled_amp.
95 comparison3 = unassembled_subimage.getDetector()[0].compareGeometry(assembled_amp)
96 self.assertTrue(comparison3 & AmplifierGeometryComparison.ASSEMBLY_DIFFERS)
97 self.assertFalse(comparison3 & AmplifierGeometryComparison.REGIONS_DIFFER)
98 else:
99 self.assertAmplifiersEqual(unassembled_subimage.getDetector()[0], unassembled_amp)
102class Ts3IngestTestCase(IngestTestBase, lsst.utils.tests.TestCase):
104 curatedCalibrationDatasetTypes = ("camera",)
105 instrumentClassName = "lsst.obs.lsst.LsstTS3"
106 ingestDir = TESTDIR
107 file = os.path.join(DATAROOT, "ts3", "raw", "2018-11-15", "201811151255111-R433-S00-det433.fits")
108 dataIds = [dict(instrument="LSST-TS3", exposure=201811151255111, detector=433)]
109 filterLabel = lsst.afw.image.FilterLabel(physical="550CutOn")
112class ComCamIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase):
114 curatedCalibrationDatasetTypes = ("camera",)
115 instrumentClassName = "lsst.obs.lsst.LsstComCam"
116 ingestDir = TESTDIR
117 file = os.path.join(DATAROOT, "comCam", "raw", "2019-05-30",
118 "3019053000001", "3019053000001-R22-S00-det000.fits")
119 dataIds = [dict(instrument="LSSTComCam", exposure=3019053000001, detector=0)]
120 filterLabel = lsst.afw.image.FilterLabel(physical="unknown", band="unknown")
123class LSSTCamIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase):
125 curatedCalibrationDatasetTypes = ("camera",)
126 instrumentClassName = "lsst.obs.lsst.LsstCam"
127 ingestDir = TESTDIR
128 file = os.path.join(DATAROOT, "lsstCam", "raw", "2019-03-19",
129 "3019031900001", "3019031900001-R10-S02-det029.fits")
130 dataIds = [dict(instrument="LSSTCam", exposure=3019031900001, detector=29)]
131 filterLabel = lsst.afw.image.FilterLabel(physical="unknown", band="unknown")
134class LSSTCamPhotodiodeIngestTestCase(lsst.utils.tests.TestCase):
135 instrumentClassName = "lsst.obs.lsst.LsstCam"
136 rawIngestTask = "lsst.obs.base.RawIngestTask"
137 ingestDir = TESTDIR
138 file = os.path.join(DATAROOT, "lsstCam", "raw", "2021-12-12",
139 "30211212000310", "30211212000310-R22-S22-det098.fits")
140 dataIds = [dict(instrument="LSSTCam", exposure=3021121200310, detector=98)]
141 filterLabel = lsst.afw.image.FilterLabel(physical="SDSSi", band="i")
142 pdPath = os.path.join(DATAROOT, "lsstCam", "raw")
144 def setUp(self):
145 """Setup for lightweight photodiode ingest task.
147 This will create the repo and register the instrument.
148 """
149 self.root = tempfile.mkdtemp(dir=self.ingestDir)
151 # Create Repo
152 runner = LogCliRunner()
153 result = runner.invoke(butlerCli, ["create", self.root])
154 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
156 # Register Instrument
157 runner = LogCliRunner()
158 result = runner.invoke(butlerCli, ["register-instrument", self.root, self.instrumentClassName])
159 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
161 def testPhotodiodeFailure(self):
162 """Test ingest to a repo missing exposure information will raise.
163 """
164 runner = LogCliRunner()
165 result = runner.invoke(
166 butlerCli,
167 [
168 "ingest-photodiode",
169 self.root,
170 self.instrumentClassName,
171 self.pdPath,
172 ],
173 )
174 self.assertEqual(result.exit_code, 1, f"output: {result.output} exception: {result.exception}")
176 def testPhotodiode(self):
177 """Test ingest to a repo with the exposure information will not raise.
178 """
179 # Ingest raw to provide exposure information.
180 outputRun = "raw_ingest_" + self.id()
181 runner = LogCliRunner()
182 result = runner.invoke(
183 butlerCli,
184 [
185 "ingest-raws",
186 self.root,
187 self.file,
188 "--output-run",
189 outputRun,
190 "--ingest-task",
191 self.rawIngestTask,
192 ],
193 )
194 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
196 # Ingest photodiode matching this exposure.
197 runner = LogCliRunner()
198 result = runner.invoke(
199 butlerCli,
200 [
201 "ingest-photodiode",
202 self.root,
203 self.instrumentClassName,
204 self.pdPath,
205 ],
206 )
207 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
209 # Confirm that we can retrieve the ingested photodiode, and
210 # that it has the correct type.
211 butler = Butler(self.root, run="LSSTCam/calib/photodiode")
212 getResult = butler.get('photodiode', dataId=self.dataIds[0])
213 self.assertIsInstance(getResult, PhotodiodeCalib)
216def setup_module(module):
217 lsst.utils.tests.init()
220if __name__ == "__main__": 220 ↛ 221line 220 didn't jump to line 221, because the condition on line 220 was never true
221 lsst.utils.tests.init()
222 unittest.main()