Coverage for tests/test_io_utils.py: 97%
194 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-25 09:00 +0000
1# This file is part of meas_extensions_scarlet.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Tests for the helpers in ``lsst.meas.extensions.scarlet.io.utils``."""
24import io
25import json
26import unittest
27import warnings
28import zipfile
30import lsst.meas.extensions.scarlet as mes
31import lsst.scarlet.lite as scl
32import lsst.utils.tests
33import numpy as np
34from lsst.afw.table import SourceCatalog, SourceTable
35from lsst.meas.extensions.scarlet.io.model_data import LsstScarletModelData
36from lsst.meas.extensions.scarlet.io.source_data import IsolatedSourceData
37from lsst.pipe.base import NoWorkFound
39import pipeline
40from scenes import SCENES
43class TestUpdateCatalogFootprints(lsst.utils.tests.TestCase):
44 """Tests for the empty-input guard in
45 ``lsst.meas.extensions.scarlet.io.updateCatalogFootprints``.
46 """
48 @staticmethod
49 def _empty_catalog():
50 # The empty-input guard returns/raises before the catalog is
51 # touched, so a bare minimal-schema catalog is enough.
52 return SourceCatalog(SourceTable.make(SourceTable.makeMinimalSchema()))
54 def test_update_catalog_footprints_empty_raises(self):
55 """``updateCatalogFootprints`` raises ``NoWorkFound`` when the
56 model data has neither blends nor isolated sources.
58 ``NoWorkFound`` is a ``lsst.pipe.base`` control-flow exception:
59 raising it short-circuits the quantum on empty input. The guard
60 previously *returned* the exception instead of raising it, so the
61 empty-input case slipped past every caller. Regression test for
62 finding C-2 of the ``audits/audit-2026-05-05.md`` audit.
63 """
64 modelData = LsstScarletModelData()
65 self.assertEqual(len(modelData.blends), 0)
66 self.assertEqual(len(modelData.isolated), 0)
68 with self.assertRaises(NoWorkFound):
69 mes.io.updateCatalogFootprints(
70 modelData, self._empty_catalog(), band="r"
71 )
73 def test_update_catalog_footprints_isolated_only_returns(self):
74 """With isolated sources but no blends, the function returns
75 ``None`` without raising — there is nothing to hydrate.
77 Pins the branch adjacent to the C-2 guard: an isolated-only
78 model (which happens for fields with only u-band images) is a
79 valid no-op, not an empty-input error.
80 """
81 isolated = {
82 1: IsolatedSourceData(
83 span_array=np.ones((3, 3), dtype=np.float32),
84 origin=(0, 0),
85 peak=(1, 1),
86 )
87 }
88 modelData = LsstScarletModelData(isolated=isolated)
89 self.assertEqual(len(modelData.blends), 0)
91 result = mes.io.updateCatalogFootprints(
92 modelData, self._empty_catalog(), band="r"
93 )
94 self.assertIsNone(result)
97class TestScarletModelToLsstScarletModel(lsst.utils.tests.TestCase):
98 """Tests for ``scarlet_model_to_lsst_scarlet_model``.
100 The converter wraps a base ``ScarletModelData`` in an
101 ``LsstScarletModelData``. Previously it dropped the input's
102 ``metadata`` and substituted ``None``, which then crashed the
103 ``_to_1_0_1`` migration on any subsequent round-trip (see finding
104 IO-3 of ``audits/audit-2026-05-05.md``).
105 """
107 @staticmethod
108 def _base_model(metadata):
109 return scl.io.ScarletModelData(blends={}, metadata=metadata)
111 def test_propagates_metadata_when_present(self):
112 """The returned ``LsstScarletModelData`` carries the same
113 ``metadata`` dict as the source, not ``None``.
114 """
115 source_metadata = {"bands": ("g", "r"), "model_psf": "placeholder"}
116 result = mes.io.utils.scarlet_model_to_lsst_scarlet_model(
117 self._base_model(source_metadata)
118 )
119 self.assertEqual(result.metadata, source_metadata)
121 def test_defaults_to_empty_dict_not_none(self):
122 """When the source's ``metadata`` is ``None``, the converter
123 substitutes an empty dict so the ``_to_1_0_1`` migration sees
124 a real dict it can call ``setdefault`` on.
125 """
126 result = mes.io.utils.scarlet_model_to_lsst_scarlet_model(
127 self._base_model(None)
128 )
129 self.assertEqual(result.metadata, {})
132class TestScarletModelDelegate(lsst.utils.tests.TestCase):
133 """Tests for
134 ``lsst.meas.extensions.scarlet.io.utils.ScarletModelDelegate.handleParameters``.
136 The delegate hooks ``LsstScarletModelData`` into Butler's
137 parameter-application pipeline. The base ``StorageClassDelegate``
138 annotates ``parameters`` as ``Mapping[str, Any] | None`` and
139 treats ``None``/``{}`` as "no parameters" — its dispatch path in
140 ``daf_butler.datastore.generic_base.post_process_get`` already
141 short-circuits with ``if assemblerParams:``, so the bug below is
142 latent in current usage, but in-memory-datastore and
143 disassembled-composite reads pass ``None``/``{}`` straight
144 through, and the delegate must agree with the base class on
145 those values.
146 """
148 @staticmethod
149 def _model_with_blends():
150 # ``handleParameters`` only inspects the keys of
151 # ``inMemoryDataset.blends`` (to slice the dict), so the
152 # values can be arbitrary sentinels — no need to build real
153 # ScarletBlendData objects.
154 return LsstScarletModelData(
155 blends={1: "blend-1", 2: "blend-2", 3: "blend-3"},
156 )
158 @staticmethod
159 def _delegate():
160 # ``StorageClassDelegate.__init__`` requires a
161 # ``StorageClass`` argument, but ``handleParameters`` never
162 # consults it; a sentinel is enough to construct the
163 # delegate in isolation from a real Butler.
164 from unittest.mock import Mock
165 return mes.io.ScarletModelDelegate(storageClass=Mock())
167 def test_handleParameters_none_returns_unchanged(self):
168 """``parameters=None`` returns the dataset unchanged.
170 Regression test for finding IO-6 of the
171 ``audits/audit-2026-05-05.md`` audit. The bug was
172 ``"blend_id" in None`` raising ``TypeError`` — the base-class
173 contract permits ``None`` and treats it as "no parameters",
174 so the delegate must do the same.
175 """
176 model = self._model_with_blends()
177 delegate = self._delegate()
179 result = delegate.handleParameters(model, None)
181 self.assertIs(result, model)
182 self.assertEqual(set(result.blends.keys()), {1, 2, 3})
184 def test_handleParameters_empty_dict_returns_unchanged(self):
185 """``parameters={}`` also returns unchanged.
187 Mirrors ``StorageClassDelegate.handleParameters`` whose
188 ``if parameters:`` guard treats the empty dict as "no
189 parameters" rather than as "unsupported parameters". The
190 pre-fix delegate raised ``ValueError("Unsupported parameters:
191 {}")`` here because the ``elif parameters is not None`` branch
192 fired on an empty dict. Companion to the ``None`` case under
193 finding IO-6 of ``audits/audit-2026-05-05.md``.
194 """
195 model = self._model_with_blends()
196 delegate = self._delegate()
198 result = delegate.handleParameters(model, {})
200 self.assertIs(result, model)
201 self.assertEqual(set(result.blends.keys()), {1, 2, 3})
203 def test_handleParameters_blend_id_filters(self):
204 """``parameters={'blend_id': ...}`` keeps only the requested
205 blends.
207 Pins the filtering branch under finding IO-6 of the
208 ``audits/audit-2026-05-05.md`` audit: the no-parameter fixes
209 above must not regress the actual partial-load path.
210 """
211 model = self._model_with_blends()
212 delegate = self._delegate()
214 result = delegate.handleParameters(model, {"blend_id": 2})
216 self.assertIs(result, model)
217 self.assertEqual(set(result.blends.keys()), {2})
218 # Iterable forms also work.
219 model = self._model_with_blends()
220 result = delegate.handleParameters(model, {"blend_id": [1, 3]})
221 self.assertEqual(set(result.blends.keys()), {1, 3})
223 def test_handleParameters_unsupported_raises(self):
224 """Non-empty parameters without ``blend_id`` raise
225 ``ValueError``.
227 Pins the rejection branch under finding IO-6 of
228 ``audits/audit-2026-05-05.md`` so a future relaxation of the
229 no-parameter case does not silently start accepting
230 unrecognized keys.
231 """
232 model = self._model_with_blends()
233 delegate = self._delegate()
235 with self.assertRaises(ValueError) as cm:
236 delegate.handleParameters(model, {"something_else": 42})
237 self.assertIn("something_else", str(cm.exception))
240class TestMonochromaticDataToScarletDeprecation(lsst.utils.tests.TestCase):
241 """Coverage retention for the deprecated
242 ``monochromaticDataToScarlet`` (scheduled for removal after v31).
244 The function is no longer called by any production code path in
245 this package, but its public-API contract is preserved for external
246 callers until removal. These tests keep the safety net by
247 exercising it directly on a synthetic blend.
248 """
250 @staticmethod
251 def _toy_blend_data():
252 # One factorized component, distinct (y, x) peak so any silent
253 # axis swap would be loud. Bands chosen so the round-trip
254 # exercises the real-band → ("dummy",) projection.
255 bands = ("g", "r", "i")
256 h, w = 6, 8
257 origin = (10, 20)
258 peak = (12, 25)
259 spectrum = np.array([1.0, 2.0, 3.0], dtype=np.float32)
260 morph = np.ones((h, w), dtype=np.float32)
261 component_data = scl.io.ScarletFactorizedComponentData(
262 origin=origin,
263 peak=peak,
264 spectrum=spectrum,
265 morph=morph,
266 )
267 source_data = scl.io.ScarletSourceData(components=[component_data])
268 blend_data = scl.io.ScarletBlendData(
269 origin=origin,
270 shape=(h, w),
271 sources={42: source_data},
272 )
273 return bands, blend_data, peak
275 def test_monochromatic_data_to_scarlet_emits_future_warning(self):
276 """``monochromaticDataToScarlet`` emits a ``FutureWarning``
277 flagging the migration to
278 ``ScarletBlendData.minimal_data_to_blend(...)[band]``.
280 Pinned so the deprecation stays visible until the function is
281 removed after v31.
282 """
283 bands, blend_data, _peak = self._toy_blend_data()
284 bbox = scl.Box(blend_data.shape, origin=blend_data.origin)
285 observation = scl.Observation.empty(
286 bands=("dummy",),
287 psfs=np.ones((1, 5, 5), dtype=np.float32),
288 model_psf=np.ones((1, 5, 5), dtype=np.float32),
289 bbox=bbox,
290 dtype=np.float32,
291 )
293 with self.assertWarns(FutureWarning):
294 blend = mes.io.monochromaticDataToScarlet(
295 blendData=blend_data, bandIndex=1, observation=observation,
296 )
298 self.assertEqual(len(blend.sources), 1)
300 def test_monochromatic_band_constants_emit_future_warning(self):
301 """Module-level ``monochromaticBand`` / ``monochromaticBands``
302 access fires a ``FutureWarning`` and resolves to the original
303 ``"dummy"`` placeholder.
305 Pinned alongside the function deprecation so the constants and
306 the function are removed together after v31.
307 """
308 from lsst.meas.extensions.scarlet.io import utils as io_utils
310 with self.assertWarns(FutureWarning):
311 self.assertEqual(io_utils.monochromaticBand, "dummy")
312 with self.assertWarns(FutureWarning):
313 self.assertEqual(io_utils.monochromaticBands, ("dummy",))
315 def test_monochromatic_data_to_scarlet_preserves_factorized_peak(self):
316 """``monochromaticDataToScarlet`` returns a source whose
317 component peak matches the persisted ``(y, x)``.
319 The deprecated function is the only remaining write-target for
320 the legacy ``("dummy",)`` per-band reconstruction; pinning the
321 peak guards against regressions in the
322 ``FactorizedComponent`` rebuild path while it lives.
323 """
324 bands, blend_data, peak = self._toy_blend_data()
325 bbox = scl.Box(blend_data.shape, origin=blend_data.origin)
326 observation = scl.Observation.empty(
327 bands=("dummy",),
328 psfs=np.ones((1, 5, 5), dtype=np.float32),
329 model_psf=np.ones((1, 5, 5), dtype=np.float32),
330 bbox=bbox,
331 dtype=np.float32,
332 )
334 with warnings.catch_warnings():
335 warnings.simplefilter("ignore", FutureWarning)
336 blend = mes.io.monochromaticDataToScarlet(
337 blendData=blend_data, bandIndex=1, observation=observation,
338 )
340 self.assertEqual(blend.sources[0].components[0].peak, peak)
343class TestLoadBlend(lsst.utils.tests.TestCase):
344 """Tests for ``loadBlend``.
346 ``loadBlend`` reconstructs a single per-blend scarlet ``Blend``
347 object from a persisted ``ScarletBlendData`` and a multiband
348 coadd. The legacy signature took ``model_psf`` directly and
349 computed per-band PSFs from the coadd; the new signature accepts
350 the full ``modelData`` and uses its already-fit PSFs verbatim,
351 which is both a more faithful round-trip and the only path that
352 works against modern persistence (legacy fields like
353 ``psf_center`` were dropped from ``ScarletBlendData`` during the
354 scarlet_lite refactor).
355 """
357 def _bundle(self):
358 # ``pipeline.deblend`` is memoized per (scene, config) so this
359 # is effectively a free lookup after the first invocation.
360 return pipeline.deblend(
361 pipeline.deconvolve(
362 pipeline.detect(pipeline.build_image(SCENES["multi-blend"]))
363 )
364 )
366 def _leaf_blend(self, modelData):
367 # Return the first leaf ``ScarletBlendData`` (i.e. the first
368 # child of the first hierarchical parent). ``loadBlend``'s
369 # contract is per-leaf, not per-parent.
370 for parent_blend in modelData.blends.values(): 370 ↛ 374line 370 didn't jump to line 374 because the loop on line 370 didn't complete
371 for child in parent_blend.children.values(): 371 ↛ 370line 371 didn't jump to line 370 because the loop on line 371 didn't complete
372 if isinstance(child, scl.io.ScarletBlendData): 372 ↛ 371line 372 didn't jump to line 371 because the condition on line 372 was always true
373 return child
374 self.fail("multi-blend scene unexpectedly produced no leaf blends")
376 def test_loadBlend_modelData_uses_metadata_model_psf(self):
377 """``loadBlend(..., modelData=...)`` builds an observation
378 whose ``model_psf`` equals ``modelData.model_psf``.
380 The previous signature derived its PSFs from the coadd at the
381 blend's ``psf_center``, which both required attributes
382 ``ScarletBlendData`` no longer carries and re-derived a PSF
383 that may not match the one used during the fit. Pinning the
384 observation's ``model_psf`` to the metadata value guards the
385 intended round-trip.
386 """
387 bundle = self._bundle()
388 modelData = bundle.result.scarletModelData
389 blendData = self._leaf_blend(modelData)
391 blend, _ = mes.io.loadBlend(
392 blendData, mCoadd=bundle.image.mCoadd, modelData=modelData,
393 )
395 np.testing.assert_array_equal(
396 blend.observation.model_psf[0],
397 modelData.model_psf,
398 )
400 def test_loadBlend_model_psf_emits_future_warning(self):
401 """Passing ``model_psf`` emits a ``FutureWarning`` flagging the
402 removal scheduled for v31.
404 The legacy path may still raise downstream (modern
405 ``ScarletBlendData`` lacks the ``psf_center`` attribute it
406 once required), so the assertion only pins the warning — any
407 post-warning exception is swallowed.
408 """
409 bundle = self._bundle()
410 modelData = bundle.result.scarletModelData
411 blendData = self._leaf_blend(modelData)
412 model_psf = modelData.model_psf
414 with self.assertWarns(FutureWarning):
415 try:
416 mes.io.loadBlend(
417 blendData,
418 model_psf=model_psf,
419 mCoadd=bundle.image.mCoadd,
420 )
421 except Exception:
422 pass
424 def test_loadBlend_requires_psf_source(self):
425 """Calling ``loadBlend`` with ``mCoadd`` but neither
426 ``modelData`` nor ``model_psf`` raises ``ValueError``.
428 Pins the precondition that some PSF source has to be passed,
429 rather than silently constructing a degenerate observation.
430 """
431 bundle = self._bundle()
432 modelData = bundle.result.scarletModelData
433 blendData = self._leaf_blend(modelData)
435 with self.assertRaises(ValueError):
436 mes.io.loadBlend(blendData, mCoadd=bundle.image.mCoadd)
438 def test_loadBlend_requires_mCoadd(self):
439 """Calling ``loadBlend`` without ``mCoadd`` raises
440 ``ValueError``.
442 ``mCoadd`` has no default value at the API level (the
443 signature uses ``None`` only to support the legacy positional
444 order); omitting it should be a loud error rather than an
445 ``AttributeError`` deep inside the observation construction.
446 """
447 bundle = self._bundle()
448 modelData = bundle.result.scarletModelData
449 blendData = self._leaf_blend(modelData)
451 with self.assertRaises(ValueError):
452 mes.io.loadBlend(blendData, modelData=modelData)
455class TestWriteScarletModelCompression(lsst.utils.tests.TestCase):
456 """Tests for compression in ``write_scarlet_model`` /
457 ``read_scarlet_model``.
459 ``write_scarlet_model`` now compresses archives with
460 ``zipfile.ZIP_DEFLATED`` by default. ``read_scarlet_model`` must
461 keep reading both the new compressed archives and older,
462 uncompressed (``ZIP_STORED``) ones, since the compression method is
463 stored per-member in each zip and decoded transparently on read.
464 """
466 def _modelData(self):
467 # ``pipeline.deblend`` is memoized per (scene, config), so this
468 # is effectively a free lookup after the first invocation.
469 bundle = pipeline.deblend(
470 pipeline.deconvolve(
471 pipeline.detect(pipeline.build_image(SCENES["multi-blend"]))
472 )
473 )
474 return bundle.result.scarletModelData
476 @staticmethod
477 def _compression_methods(buf):
478 # Return the set of per-member compression methods used in the
479 # archive held by ``buf``.
480 buf.seek(0)
481 with zipfile.ZipFile(buf, "r") as zf:
482 return {info.compress_type for info in zf.infolist()}
484 def _assert_models_equal(self, modelData1, modelData2):
485 # ``as_dict`` captures the whole model (metadata, every blend
486 # and source, and the isolated sources). ``json.dumps`` with
487 # sorted keys gives a canonical form and stringifies the int
488 # source-keys, which is the one non-value difference a JSON
489 # round-trip introduces. The numeric fields round-trip
490 # bitwise-identical, so this comparison is exact.
491 self.assertEqual(
492 json.dumps(modelData1.as_dict(), sort_keys=True),
493 json.dumps(modelData2.as_dict(), sort_keys=True),
494 )
496 def test_write_defaults_to_deflated(self):
497 """By default every member of the archive is DEFLATE-compressed.
499 Pins the "always compress going forward" contract: a fresh
500 ``write_scarlet_model`` must not leave any ``ZIP_STORED``
501 members behind.
502 """
503 modelData = self._modelData()
504 buf = io.BytesIO()
505 mes.io.utils.write_scarlet_model(buf, modelData)
507 methods = self._compression_methods(buf)
508 self.assertEqual(methods, {zipfile.ZIP_DEFLATED})
510 def test_compressed_roundtrip(self):
511 """A default (compressed) write round-trips through
512 ``read_scarlet_model``.
513 """
514 modelData = self._modelData()
515 buf = io.BytesIO()
516 mes.io.utils.write_scarlet_model(buf, modelData)
518 buf.seek(0)
519 modelData2 = mes.io.utils.read_scarlet_model(buf)
520 self._assert_models_equal(modelData, modelData2)
522 def test_reads_legacy_uncompressed_archive(self):
523 """A ``ZIP_STORED`` (uncompressed) archive still reads.
525 Older files were written without a compression method; the read
526 path must remain backward compatible.
527 """
528 modelData = self._modelData()
529 buf = io.BytesIO()
530 # Explicitly write an uncompressed archive to emulate a
531 # pre-compression file on disk.
532 mes.io.utils.write_scarlet_model(buf, modelData, compression=zipfile.ZIP_STORED)
534 self.assertEqual(self._compression_methods(buf), {zipfile.ZIP_STORED})
536 buf.seek(0)
537 modelData2 = mes.io.utils.read_scarlet_model(buf)
538 self._assert_models_equal(modelData, modelData2)
540 def test_compression_reduces_size(self):
541 """The compressed archive is smaller than the uncompressed one.
543 A cheap sanity check that compression is actually being applied
544 rather than merely tagged.
545 """
546 modelData = self._modelData()
548 stored = io.BytesIO()
549 mes.io.utils.write_scarlet_model(stored, modelData, compression=zipfile.ZIP_STORED)
551 deflated = io.BytesIO()
552 mes.io.utils.write_scarlet_model(deflated, modelData)
554 self.assertLess(len(deflated.getvalue()), len(stored.getvalue()))
557def setup_module(module):
558 lsst.utils.tests.init()
561class MemoryTester(lsst.utils.tests.MemoryTestCase):
562 pass
565if __name__ == "__main__": 565 ↛ 566line 565 didn't jump to line 566 because the condition on line 565 was never true
566 lsst.utils.tests.init()
567 unittest.main()