From dd25c63cd7f14c715f87ced85a80bf001b7e1fe7 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 23 Oct 2023 11:02:08 +0200 Subject: [PATCH 001/118] Increment version --- setup.py | 2 +- src/isal/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9bc284fd..1cb13991 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ def build_isa_l(): setup( name="isal", - version="1.5.1", + version="1.6.0-dev", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index 9306adfc..f7257098 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -27,4 +27,4 @@ "__version__" ] -__version__ = "1.5.1" +__version__ = "1.6.0-dev" From 367745ac3f5403cc6bc8b7c61882e53c6042da28 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 23 Oct 2023 13:42:34 +0200 Subject: [PATCH 002/118] Fix readthedocs build configuration --- .readthedocs.yml | 17 +++++++++++++---- docs/conda-environment.yml | 12 ------------ 2 files changed, 13 insertions(+), 16 deletions(-) delete mode 100644 docs/conda-environment.yml diff --git a/.readthedocs.yml b/.readthedocs.yml index 2866edaf..910b7650 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,7 +3,16 @@ formats: [] # Do not build epub and pdf python: install: - - method: pip - path: . -conda: - environment: docs/conda-environment.yml \ No newline at end of file + - requirements: "requirements-docs.txt" + - method: "pip" + path: "." + +sphinx: + configuration: docs/conf.py + +build: + os: "ubuntu-22.04" + tools: + python: "3" + apt_packages: + - libisal-dev diff --git a/docs/conda-environment.yml b/docs/conda-environment.yml deleted file mode 100644 index a23cb482..00000000 --- a/docs/conda-environment.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: rtd -channels: - - conda-forge - - defaults -dependencies: - - isa-l - - python >=3.6 - - sphinx - - setuptools - - pip: - - sphinx-rtd-theme - - sphinx-argparse \ No newline at end of file From 4afcfaa4d8dbbd79d4a65abe17520da7eaca16ef Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Oct 2023 19:51:10 +0200 Subject: [PATCH 003/118] Add asan environment --- tox.ini | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tox.ini b/tox.ini index c94c6690..7e74d40a 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,15 @@ commands = coverage html -i coverage xml -i +[testenv:asan] +setenv= + PYTHONDEVMODE=1 + PYTHONMALLOC=malloc + CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer +allowlist_externals=bash +commands= + bash -c 'LD_PRELOAD=$(find /usr/lib -name "libasan.so*" | head -n 1) pytest tests' + [testenv:compliance] deps=pytest commands= From b78ef86d837128d4ce65dc997aa8a20e72c2b483 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Thu, 26 Oct 2023 08:22:46 +0200 Subject: [PATCH 004/118] Add missing decref when error is thrown in decompressobj --- src/isal/isal_zlibmodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 257260f8..41971445 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -793,6 +793,7 @@ isal_zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict) err = wbits_to_flag_and_hist_bits_inflate(wbits, &hist_bits, &flag); if (err < 0) { PyErr_Format(PyExc_ValueError, "Invalid wbits value: %d", wbits); + Py_DECREF(self); return NULL; } else if (err == 0) { From 47da5dbae7f3e00accccbea9f74d6c9a1793e14a Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Oct 2023 07:11:36 +0200 Subject: [PATCH 005/118] Properly decref magic bytes representation --- src/isal/isal_zlibmodule.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 41971445..2c5c8a92 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1684,9 +1684,11 @@ GzipReader_read_into_buffer(GzipReader *self, uint8_t *out_buffer, size_t out_bu if (!(magic1 == 0x1f && magic2 == 0x8b)) { Py_BLOCK_THREADS; + PyObject *magic_obj = PyBytes_FromStringAndSize((char *)current_pos, 2); PyErr_Format(BadGzipFile, "Not a gzipped file (%R)", - PyBytes_FromStringAndSize((char *)current_pos, 2)); + magic_obj); + Py_DECREF(magic_obj); return -1; }; uint8_t method = current_pos[2]; From 9a27be448d79f8cafa20ccc27b64a58e9ef3d68f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Oct 2023 07:36:23 +0200 Subject: [PATCH 006/118] Also build with address sanitizer --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 338a177c..6eb1cd64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: tox_env: - docs - twine_check + - asan runs-on: ubuntu-latest steps: - uses: actions/checkout@v2.3.4 From a259b509c43d3d4a42b2d9be6695ce97673119af Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Oct 2023 07:40:12 +0200 Subject: [PATCH 007/118] Make sure libasan is installed --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6eb1cd64..6bd63f45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: with: python-version: 3.8 - name: Install isal - run: sudo apt-get install libisal-dev + run: sudo apt-get install libisal-dev libasan8 - name: Install tox and upgrade setuptools and pip run: pip install --upgrade tox setuptools pip - name: Run tox -e ${{ matrix.tox_env }} From 0267edc2069a98e0c9d49796313e7b0ad18a4e3a Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Oct 2023 07:50:55 +0200 Subject: [PATCH 008/118] Make test a bit more debuggable on the CI --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 7e74d40a..a42fde97 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,7 @@ setenv= CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer allowlist_externals=bash commands= - bash -c 'LD_PRELOAD=$(find /usr/lib -name "libasan.so*" | head -n 1) pytest tests' + bash -c 'export LD_PRELOAD=$(find /usr/lib -name "libasan.so*" | head -n 1) && printenv LD_PRELOAD && pytest tests' [testenv:compliance] deps=pytest From 42e5ba28d5de7167290774003e1931c24928e3e7 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Oct 2023 07:55:20 +0200 Subject: [PATCH 009/118] First import in order to check errors --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a42fde97..52f46a39 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,7 @@ setenv= CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer allowlist_externals=bash commands= - bash -c 'export LD_PRELOAD=$(find /usr/lib -name "libasan.so*" | head -n 1) && printenv LD_PRELOAD && pytest tests' + bash -c 'export LD_PRELOAD=$(find /usr/lib -name "libasan.so*" | head -n 1) && printenv LD_PRELOAD && python -c "from isal import isal_zlib" && pytest tests' [testenv:compliance] deps=pytest From fe6d7596f9617efdaf481ca749ecdda67158a533 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Oct 2023 08:02:42 +0200 Subject: [PATCH 010/118] Separate asan structure --- .github/workflows/ci.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bd63f45..8f6a3851 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,6 @@ jobs: tox_env: - docs - twine_check - - asan runs-on: ubuntu-latest steps: - uses: actions/checkout@v2.3.4 @@ -45,7 +44,7 @@ jobs: with: python-version: 3.8 - name: Install isal - run: sudo apt-get install libisal-dev libasan8 + run: sudo apt-get install libisal-dev - name: Install tox and upgrade setuptools and pip run: pip install --upgrade tox setuptools pip - name: Run tox -e ${{ matrix.tox_env }} @@ -100,6 +99,23 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v1 + test-asan: + runs-on: "ubuntu-latest" + steps: + - uses: actions/checkout@v2.3.4 + with: + submodules: recursive + - name: Set up Python 3.8 + uses: actions/setup-python@v2.2.1 + with: + python-version: "3.8" + - name: Install tox and upgrade setuptools + run: pip install --upgrade tox setuptools + - name: Install build dependencies (Linux) # Yasm in pypa/manylinux images. + run: sudo apt install nasm + - name: Run asan tests + run: tox -e asan + test-arch: if: startsWith(github.ref, 'refs/tags') || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main' runs-on: "ubuntu-latest" From 3fbc980f409dae111fc97c0a8f90244fe2105e6f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Oct 2023 08:06:36 +0200 Subject: [PATCH 011/118] Revert github ci changes --- .github/workflows/ci.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f6a3851..338a177c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,23 +99,6 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v1 - test-asan: - runs-on: "ubuntu-latest" - steps: - - uses: actions/checkout@v2.3.4 - with: - submodules: recursive - - name: Set up Python 3.8 - uses: actions/setup-python@v2.2.1 - with: - python-version: "3.8" - - name: Install tox and upgrade setuptools - run: pip install --upgrade tox setuptools - - name: Install build dependencies (Linux) # Yasm in pypa/manylinux images. - run: sudo apt install nasm - - name: Run asan tests - run: tox -e asan - test-arch: if: startsWith(github.ref, 'refs/tags') || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main' runs-on: "ubuntu-latest" From 114d4f4d762bc5d9aa39ab5ce1ea7247c1bb0f58 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Oct 2023 11:39:17 +0200 Subject: [PATCH 012/118] Make ASAN checks of release procedure --- .github/release_checklist.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/release_checklist.md b/.github/release_checklist.md index c47123ba..7093fe46 100644 --- a/.github/release_checklist.md +++ b/.github/release_checklist.md @@ -5,6 +5,7 @@ Release checklist - [ ] Set version to a stable number. - [ ] Change current development version in `CHANGELOG.rst` to stable version. - [ ] Change the version in `__init__.py` +- [ ] Check if the address sanitizer does not find any problems using `tox -e asan` - [ ] Merge the release branch into `main`. - [ ] Created an annotated tag with the stable version number. Include changes from CHANGELOG.rst. From bab10a9f952afae8d6dbba78d745b3b18823ca36 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Oct 2023 11:41:20 +0200 Subject: [PATCH 013/118] Add memory leak fixes to changelog --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0f1eaff3..d51e515b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 1.6.0-dev +----------------- ++ Fix a memory leak that occurred when an error was thrown for a gzip header + with the wrong magic numbers. ++ Fix a memory leak that occurred when isal_zlib.decompressobj was given a + wrong wbits value. + version 1.5.1 ----------------- + Fix a memory leak in the GzipReader.readall implementation. From e6778171988fdca1b40d14afafc8cbee7b0f0461 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 27 Oct 2023 17:04:24 +0200 Subject: [PATCH 014/118] Use gcc command rather than find --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 52f46a39..256ee95e 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,7 @@ setenv= CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer allowlist_externals=bash commands= - bash -c 'export LD_PRELOAD=$(find /usr/lib -name "libasan.so*" | head -n 1) && printenv LD_PRELOAD && python -c "from isal import isal_zlib" && pytest tests' + bash -c 'export LD_PRELOAD=$(gcc -print-file-name=libasan.so) && printenv LD_PRELOAD && python -c "from isal import isal_zlib" && pytest tests' [testenv:compliance] deps=pytest From c84f90d3302db1128a8dd651afa9de96f16c0e1c Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 3 Nov 2023 15:58:52 +0100 Subject: [PATCH 015/118] Reproduce resourcewarning --- tests/test_igzip_threaded.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index a0f581c6..bd1282c2 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -139,10 +139,11 @@ def test_writer_not_readable(): def test_writer_wrong_level(): - with pytest.raises(ValueError) as error: - igzip_threaded._ThreadedGzipWriter(io.BytesIO(), level=42) - error.match("Invalid compression level") - error.match("42") + with tempfile.NamedTemporaryFile("wb") as tmp: + with pytest.raises(ValueError) as error: + igzip_threaded.open(tmp.name, mode="wb", compresslevel=42) + error.match("Invalid compression level") + error.match("42") def test_writer_too_low_threads(): From 54ebf53379c346b2bbec5cb4cfcd94eb5749d5cf Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 3 Nov 2023 16:03:58 +0100 Subject: [PATCH 016/118] Prevent unclosed filehandles by moving opening logic after other tests --- src/isal/igzip_threaded.py | 30 +++++++++++++++++------------- tests/test_igzip_threaded.py | 2 +- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index 20142e7a..b42a8186 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -57,19 +57,13 @@ def open(filename, mode="rb", compresslevel=igzip._COMPRESS_LEVEL_TRADEOFF, except: # noqa: E722 threads = 1 open_mode = mode.replace("t", "b") - if isinstance(filename, (str, bytes)) or hasattr(filename, "__fspath__"): - binary_file = builtins.open(filename, open_mode) - elif hasattr(filename, "read") or hasattr(filename, "write"): - binary_file = filename - else: - raise TypeError("filename must be a str or bytes object, or a file") if "r" in mode: gzip_file = io.BufferedReader( - _ThreadedGzipReader(binary_file, block_size=block_size)) + _ThreadedGzipReader(filename, block_size=block_size)) else: gzip_file = io.BufferedWriter( _ThreadedGzipWriter( - fp=binary_file, + filename, block_size=block_size, level=compresslevel, threads=threads @@ -81,10 +75,20 @@ def open(filename, mode="rb", compresslevel=igzip._COMPRESS_LEVEL_TRADEOFF, return gzip_file +def open_as_binary_stream(filename, open_mode): + if isinstance(filename, (str, bytes)) or hasattr(filename, "__fspath__"): + binary_file = builtins.open(filename, open_mode) + elif hasattr(filename, "read") or hasattr(filename, "write"): + binary_file = filename + else: + raise TypeError("filename must be a str or bytes object, or a file") + return binary_file + + class _ThreadedGzipReader(io.RawIOBase): - def __init__(self, fp, queue_size=2, block_size=1024 * 1024): - self.raw = fp - self.fileobj = igzip._IGzipReader(fp, buffersize=8 * block_size) + def __init__(self, filename, queue_size=2, block_size=1024 * 1024): + self.raw = open_as_binary_stream(filename, "rb") + self.fileobj = igzip._IGzipReader(self.raw, buffersize=8 * block_size) self.pos = 0 self.read_file = False self.queue = queue.Queue(queue_size) @@ -193,7 +197,7 @@ class _ThreadedGzipWriter(io.RawIOBase): compressing and output is handled in one thread. """ def __init__(self, - fp: BinaryIO, + filename, level: int = isal_zlib.ISAL_DEFAULT_COMPRESSION, threads: int = 1, queue_size: int = 1, @@ -201,7 +205,6 @@ def __init__(self, ): self.lock = threading.Lock() self.exception: Optional[Exception] = None - self.raw = fp self.level = level self.previous_block = b"" # Deflating random data results in an output a little larger than the @@ -236,6 +239,7 @@ def __init__(self, self.running = False self._size = 0 self._closed = False + self.raw = open_as_binary_stream(filename, "wb") self._write_gzip_header() self.start() diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index bd1282c2..be6c3ca4 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -99,7 +99,7 @@ def test_threaded_write_oversized_block_no_error(threads): @pytest.mark.parametrize("threads", [1, 3]) def test_threaded_write_error(threads): f = igzip_threaded._ThreadedGzipWriter( - fp=io.BytesIO(), level=3, + io.BytesIO(), level=3, threads=threads, block_size=8 * 1024) # Bypass the write method which should not allow blocks larger than # block_size. From fbe1eb300876e566c50cd9804b0b8900c6b1ab40 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 3 Nov 2023 16:04:43 +0100 Subject: [PATCH 017/118] Fix linter issues --- src/isal/igzip_threaded.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index b42a8186..dd34e704 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -12,7 +12,7 @@ import queue import struct import threading -from typing import BinaryIO, List, Optional, Tuple +from typing import List, Optional, Tuple from . import igzip, isal_zlib @@ -56,7 +56,6 @@ def open(filename, mode="rb", compresslevel=igzip._COMPRESS_LEVEL_TRADEOFF, threads = multiprocessing.cpu_count() except: # noqa: E722 threads = 1 - open_mode = mode.replace("t", "b") if "r" in mode: gzip_file = io.BufferedReader( _ThreadedGzipReader(filename, block_size=block_size)) From 27f3f7bdb8b2edbb5841d34fd0aa1ca3dadbe26d Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 3 Nov 2023 16:09:04 +0100 Subject: [PATCH 018/118] Add bugfix to changelog --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d51e515b..65eab10e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,8 @@ Changelog version 1.6.0-dev ----------------- ++ Fix a bug where a filehandle remained opened when ``igzip_threaded.open`` + was used for writing with a wrong compression level. + Fix a memory leak that occurred when an error was thrown for a gzip header with the wrong magic numbers. + Fix a memory leak that occurred when isal_zlib.decompressobj was given a From b158a268ea96f512938c472acadd053d5c12e065 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 3 Nov 2023 16:30:58 +0100 Subject: [PATCH 019/118] Set stable version --- CHANGELOG.rst | 2 +- setup.py | 2 +- src/isal/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 65eab10e..3e936f62 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. -version 1.6.0-dev +version 1.5.2 ----------------- + Fix a bug where a filehandle remained opened when ``igzip_threaded.open`` was used for writing with a wrong compression level. diff --git a/setup.py b/setup.py index 1cb13991..22c16742 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ def build_isa_l(): setup( name="isal", - version="1.6.0-dev", + version="1.5.2", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index f7257098..8a39a81a 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -27,4 +27,4 @@ "__version__" ] -__version__ = "1.6.0-dev" +__version__ = "1.5.2" From 60470b9273f41dc6a50f29c18c19669fed7baf49 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 3 Nov 2023 16:43:59 +0100 Subject: [PATCH 020/118] Set new version --- setup.py | 2 +- src/isal/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 22c16742..1cb13991 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ def build_isa_l(): setup( name="isal", - version="1.5.2", + version="1.6.0-dev", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index 8a39a81a..f7257098 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -27,4 +27,4 @@ "__version__" ] -__version__ = "1.5.2" +__version__ = "1.6.0-dev" From e55dddd92fa9c2dcd7b021181fb7a73552b88f79 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 24 Nov 2023 09:16:12 +0100 Subject: [PATCH 021/118] Fix append mode bug --- CHANGELOG.rst | 5 +++++ src/isal/igzip_threaded.py | 8 +++++++- tests/test_igzip_threaded.py | 22 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3e936f62..de602380 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 1.5.3 +----------------- ++ Fix a bug where append mode would not work when using + ``igzip_threaded.open``. + version 1.5.2 ----------------- + Fix a bug where a filehandle remained opened when ``igzip_threaded.open`` diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index dd34e704..99eafc17 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -63,6 +63,7 @@ def open(filename, mode="rb", compresslevel=igzip._COMPRESS_LEVEL_TRADEOFF, gzip_file = io.BufferedWriter( _ThreadedGzipWriter( filename, + mode.replace("t", "b"), block_size=block_size, level=compresslevel, threads=threads @@ -197,11 +198,16 @@ class _ThreadedGzipWriter(io.RawIOBase): """ def __init__(self, filename, + mode: str = "wb", level: int = isal_zlib.ISAL_DEFAULT_COMPRESSION, threads: int = 1, queue_size: int = 1, block_size: int = 1024 * 1024, ): + if "t" in mode or "r" in mode: + raise ValueError("Only binary writing is supported") + if "b" not in mode: + mode += "b" self.lock = threading.Lock() self.exception: Optional[Exception] = None self.level = level @@ -238,7 +244,7 @@ def __init__(self, self.running = False self._size = 0 self._closed = False - self.raw = open_as_binary_stream(filename, "wb") + self.raw = open_as_binary_stream(filename, mode) self._write_gzip_header() self.start() diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index be6c3ca4..dbee85f8 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -169,3 +169,25 @@ def test_writer_write_after_close(threads): with pytest.raises(ValueError) as error: f.write(b"abc") error.match("closed") + + +def test_igzip_threaded_append(tmp_path): + test_file = tmp_path / "test.txt.gz" + with igzip_threaded.open(test_file, "wb") as f: + f.write(b"AB") + with igzip_threaded.open(test_file, mode="ab") as f: + f.write(b"CD") + with gzip.open(test_file, "rb") as f: + contents = f.read() + assert contents == b"ABCD" + + +def test_igzip_threaded_append_text_mode(tmp_path): + test_file = tmp_path / "test.txt.gz" + with igzip_threaded.open(test_file, "wt") as f: + f.write("AB") + with igzip_threaded.open(test_file, mode="at") as f: + f.write("CD") + with gzip.open(test_file, "rt") as f: + contents = f.read() + assert contents == "ABCD" From 756f0978844dd02daebf95f21e9584eb84d8df70 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 24 Nov 2023 09:27:47 +0100 Subject: [PATCH 022/118] Update version number, add threads keyword --- setup.py | 4 ++-- src/isal/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 1cb13991..7d422418 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ def build_isa_l(): setup( name="isal", - version="1.6.0-dev", + version="1.5.3", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", @@ -145,7 +145,7 @@ def build_isa_l(): long_description_content_type="text/x-rst", cmdclass={"build_ext": BuildIsalExt}, license="PSF-2.0", - keywords="isal isa-l compression deflate gzip igzip", + keywords="isal isa-l compression deflate gzip igzip threads", zip_safe=False, packages=find_packages('src'), package_dir={'': 'src'}, diff --git a/src/isal/__init__.py b/src/isal/__init__.py index f7257098..36c2a468 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -27,4 +27,4 @@ "__version__" ] -__version__ = "1.6.0-dev" +__version__ = "1.5.3" From a7d017150cfb614a40ac440ef7f6bd7848b429ab Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 24 Nov 2023 09:34:44 +0100 Subject: [PATCH 023/118] Update version --- setup.py | 2 +- src/isal/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7d422418..728200be 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ def build_isa_l(): setup( name="isal", - version="1.5.3", + version="1.6.0-dev", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index 36c2a468..f7257098 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -27,4 +27,4 @@ "__version__" ] -__version__ = "1.5.3" +__version__ = "1.6.0-dev" From dc4b5c33e4b68cf4f1dc0cf639eac188f9b83f57 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Sun, 24 Dec 2023 10:36:41 +0100 Subject: [PATCH 024/118] Prevent operations on raw file before it is successfully opened if error is thrown during __init__ --- src/isal/igzip_threaded.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index 99eafc17..adb2c24d 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -204,6 +204,9 @@ def __init__(self, queue_size: int = 1, block_size: int = 1024 * 1024, ): + # File should be closed during init, so __exit__ method does not + # touch the self.raw value before it is initialized. + self._closed = True if "t" in mode or "r" in mode: raise ValueError("Only binary writing is supported") if "b" not in mode: @@ -243,8 +246,8 @@ def __init__(self, self._crc = 0 self.running = False self._size = 0 - self._closed = False self.raw = open_as_binary_stream(filename, mode) + self._closed = False self._write_gzip_header() self.start() From 4deee8f9ae7408f086e740dd93b8b0e213076221 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 5 Jan 2024 09:31:42 +0100 Subject: [PATCH 025/118] Add bugfix to changelog --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index de602380..c6e57dba 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 1.6.0-dev +----------------- ++ Fix an error that occurred in the ``__close__`` function when a threaded + writer was initialized with incorrect parameters. + version 1.5.3 ----------------- + Fix a bug where append mode would not work when using From a5817be36bd7e4740da02d7d551ecaa9d7dd15d8 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 5 Jan 2024 09:27:32 +0100 Subject: [PATCH 026/118] Use latest isa-l --- .gitmodules | 2 +- src/isal/isa-l | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 11a84d8b..d79a05a7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "src/isal/isa-l"] path = src/isal/isa-l - url = https://github.com/rhpvorderman/isa-l.git + url = https://github.com/intel/isa-l.git diff --git a/src/isal/isa-l b/src/isal/isa-l index d80c10d6..29d99fce 160000 --- a/src/isal/isa-l +++ b/src/isal/isa-l @@ -1 +1 @@ -Subproject commit d80c10d68aa42ab90dcb1938f8cace21d09faf0a +Subproject commit 29d99fce2631c435b37b6c6842aa18da41ce83a6 From e0125c954ea0d6274c755eebd11e1ea1ba79d49a Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 10 Jan 2024 09:44:07 +0100 Subject: [PATCH 027/118] Update to latest version of ISA-L before release of 2.31 --- src/isal/isa-l | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isal/isa-l b/src/isal/isa-l index 29d99fce..455fdded 160000 --- a/src/isal/isa-l +++ b/src/isal/isa-l @@ -1 +1 @@ -Subproject commit 29d99fce2631c435b37b6c6842aa18da41ce83a6 +Subproject commit 455fdded4e2c1e658a74005d2cdc66ff1649b78d From 4b617344517ef524abea58231bfc47612e423d0b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 17 Jan 2024 12:48:13 +0100 Subject: [PATCH 028/118] Update ISA-L to frozen code --- src/isal/isa-l | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isal/isa-l b/src/isal/isa-l index 455fdded..d4e1c21a 160000 --- a/src/isal/isa-l +++ b/src/isal/isa-l @@ -1 +1 @@ -Subproject commit 455fdded4e2c1e658a74005d2cdc66ff1649b78d +Subproject commit d4e1c21acbc61df30804004b61b068d9da7d3c6a From 40267e1531da76d5ce4a1e43782374c074c1cd90 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 24 Jan 2024 10:46:15 +0100 Subject: [PATCH 029/118] Use ISA-L 2.31.0 --- CHANGELOG.rst | 1 + src/isal/isa-l | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c6e57dba..39bcbb53 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,7 @@ Changelog version 1.6.0-dev ----------------- ++ Update statically linked ISA-L release to 2.31.0 + Fix an error that occurred in the ``__close__`` function when a threaded writer was initialized with incorrect parameters. diff --git a/src/isal/isa-l b/src/isal/isa-l index d4e1c21a..bd226375 160000 --- a/src/isal/isa-l +++ b/src/isal/isa-l @@ -1 +1 @@ -Subproject commit d4e1c21acbc61df30804004b61b068d9da7d3c6a +Subproject commit bd226375027899087bd48f3e59b910430615cc0a From a7ffab76af5c049ceacc1fd0b90d0816115d9933 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 24 Jan 2024 10:51:41 +0100 Subject: [PATCH 030/118] Assume latest version with Windows header and zlib patch is available --- setup.py | 11 +++-------- src/isal/__init__.py | 15 ++------------- tests/test_compat.py | 5 ----- tests/test_zlib_compliance.py | 4 ---- 4 files changed, 5 insertions(+), 30 deletions(-) diff --git a/setup.py b/setup.py index 728200be..008e16ba 100644 --- a/setup.py +++ b/setup.py @@ -33,16 +33,11 @@ DEFAULT_CACHE_FILE)) EXTENSIONS = [ - Extension("isal.isal_zlib", ["src/isal/isal_zlibmodule.c"]), - Extension("isal.igzip_lib", ["src/isal/igzip_libmodule.c"]), + Extension("isal.isal_zlib", ["src/isal/isal_zlibmodule.c"]), + Extension("isal.igzip_lib", ["src/isal/igzip_libmodule.c"]), + Extension("isal._isal", ["src/isal/_isalmodule.c"]), ] -# This does not add the extension on windows for dynamic linking. The required -# header file might be missing. -if not (SYSTEM_IS_WINDOWS and - os.getenv("PYTHON_ISAL_LINK_DYNAMIC") is not None): - EXTENSIONS.append(Extension("isal._isal", ["src/isal/_isalmodule.c"])) - class BuildIsalExt(build_ext): def build_extension(self, ext): diff --git a/src/isal/__init__.py b/src/isal/__init__.py index f7257098..d6c02fdb 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -5,19 +5,8 @@ # This file is part of python-isal which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. -from typing import Optional - -try: - from . import _isal - ISAL_MAJOR_VERSION: Optional[int] = _isal.ISAL_MAJOR_VERSION - ISAL_MINOR_VERSION: Optional[int] = _isal.ISAL_MINOR_VERSION - ISAL_PATCH_VERSION: Optional[int] = _isal.ISAL_PATCH_VERSION - ISAL_VERSION: Optional[str] = _isal.ISAL_VERSION -except ImportError: - ISAL_MAJOR_VERSION = None - ISAL_MINOR_VERSION = None - ISAL_PATCH_VERSION = None - ISAL_VERSION = None +from ._isal import (ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION, + ISAL_VERSION) __all__ = [ "ISAL_MAJOR_VERSION", diff --git a/tests/test_compat.py b/tests/test_compat.py index baa86bfe..0ea54f70 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -11,7 +11,6 @@ import gzip import itertools -import os import zlib from pathlib import Path @@ -35,8 +34,6 @@ # Wbits for ZLIB compression, GZIP compression, and RAW compressed streams WBITS_RANGE = list(range(9, 16)) + list(range(25, 32)) + list(range(-15, -8)) -DYNAMICALLY_LINKED = os.getenv("PYTHON_ISAL_LINK_DYNAMIC") is not None - @pytest.mark.parametrize(["data_size", "value"], itertools.product(DATA_SIZES, SEEDS)) @@ -93,8 +90,6 @@ def test_decompress_isal_zlib(data_size, level): @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel"], itertools.product([128 * 1024], range(4), WBITS_RANGE, range(1, 10))) -@pytest.mark.xfail(condition=DYNAMICALLY_LINKED, - reason="Dynamically linked version may not have patch.") def test_compress_compressobj(data_size, level, wbits, memLevel): data = DATA[:data_size] compressobj = isal_zlib.compressobj(level=level, diff --git a/tests/test_zlib_compliance.py b/tests/test_zlib_compliance.py index 4e2285b4..e2270050 100644 --- a/tests/test_zlib_compliance.py +++ b/tests/test_zlib_compliance.py @@ -40,10 +40,6 @@ class VersionTestCase(unittest.TestCase): - - @unittest.skipIf(os.getenv("PYTHON_ISAL_LINK_DYNAMIC") is not None and - sys.platform.startswith("win"), - "Header file missing on windows") def test_library_version(self): # Test that the major version of the actual library in use matches the # major version that we were compiled against. We can't guarantee that From 06e62af5c4abc63292157ba37df2293d408a32b6 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 24 Jan 2024 12:18:48 +0100 Subject: [PATCH 031/118] Test on latest python 3.13 --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 338a177c..5193c14a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,7 @@ jobs: - "3.10" - "3.11" - "3.12" + - "3.13-dev" - "pypy-3.9" - "pypy-3.10" os: ["ubuntu-latest"] From 694059a189ff24d112f91ecb0b9fa279f828fa28 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 24 Jan 2024 12:21:49 +0100 Subject: [PATCH 032/118] Do not use private _PyBytes_Join API call --- src/isal/isal_zlibmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 2c5c8a92..4ba540ec 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -2017,7 +2017,7 @@ GzipReader_readall(GzipReader *self, PyObject *Py_UNUSED(ignore)) Py_DECREF(chunk_list); return NULL; } - PyObject *ret = _PyBytes_Join(empty_bytes, chunk_list); + PyObject *ret = PyObject_CallMethod(empty_bytes, "join", "O", chunk_list); Py_DECREF(empty_bytes); Py_DECREF(chunk_list); return ret; From 9b2556a26bf4dbfd1f07c91a4024ea4d28b96cbd Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 19 Feb 2024 08:47:28 +0100 Subject: [PATCH 033/118] Add tests to reproduce bug where openening with a wrong compresslevel in read mode causes a crash. --- tests/test_gzip_compliance.py | 14 ++++++++++++++ tests/test_igzip_threaded.py | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index 8439610b..cc979317 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -456,6 +456,13 @@ def test_with_open(self): else: self.fail("1/0 didn't raise an exception") + def test_read_and_compresslevel(self): + with igzip.GzipFile(self.filename, "wb") as f: + f.write(b"xxx") + with igzip.GzipFile(self.filename, "rb", compresslevel=17) as f: + data = f.read() + assert data == b"xxx" + def test_zero_padded_file(self): with igzip.GzipFile(self.filename, "wb") as f: f.write(data1 * 50) @@ -785,6 +792,13 @@ def test_newline(self): with igzip.open(self.filename, "rt", newline="\r") as f: self.assertEqual(f.readlines(), [uncompressed]) + def test_reading_and_compresslevel(self): + with igzip.open(self.filename, "wb") as f: + f.write(data1) + with igzip.open(self.filename, "rb", compresslevel=17) as f: + text = f.read() + assert text == data1 + def create_and_remove_directory(directory): def decorator(function): diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index dbee85f8..dae40a41 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -191,3 +191,11 @@ def test_igzip_threaded_append_text_mode(tmp_path): with gzip.open(test_file, "rt") as f: contents = f.read() assert contents == "ABCD" + + +def test_igzip_threaded_open_compresslevel_and_reading(tmp_path): + test_file = tmp_path / "test.txt.gz" + test_file.write_bytes(gzip.compress(b"thisisatest")) + with igzip_threaded.open(test_file, compresslevel=5) as f: + text = f.read() + assert text == b"thisisatest" From be6bdc7283a437752c54e1c909ccad78305595f8 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 19 Feb 2024 08:53:17 +0100 Subject: [PATCH 034/118] Only throw compression level errors in write and append modes --- CHANGELOG.rst | 1 + src/isal/igzip.py | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 39bcbb53..a248d5ee 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,7 @@ Changelog version 1.6.0-dev ----------------- ++ Fix a bug where compression levels for IGzipFile where checked in read mode. + Update statically linked ISA-L release to 2.31.0 + Fix an error that occurred in the ``__close__`` function when a threaded writer was initialized with incorrect parameters. diff --git a/src/isal/igzip.py b/src/isal/igzip.py index 5b942862..3c2ed2cb 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -151,11 +151,12 @@ def __init__(self, filename=None, mode=None, If omitted or None, the current time is used. """ if not (isal_zlib.ISAL_BEST_SPEED <= compresslevel - <= isal_zlib.ISAL_BEST_COMPRESSION): + <= isal_zlib.ISAL_BEST_COMPRESSION) and "r" not in mode: raise ValueError( - "Compression level should be between {0} and {1}.".format( - isal_zlib.ISAL_BEST_SPEED, isal_zlib.ISAL_BEST_COMPRESSION - )) + f"Compression level should be between " + f"{isal_zlib.ISAL_BEST_SPEED} and " + f"{isal_zlib.ISAL_BEST_COMPRESSION}, got {compresslevel}." + ) super().__init__(filename, mode, compresslevel, fileobj, mtime) if self.mode == WRITE: self.compress = isal_zlib.compressobj(compresslevel, From 015c719e3fed3947e1d9a55b879c23e8c40998a5 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 19 Feb 2024 09:29:33 +0100 Subject: [PATCH 035/118] Set stable version --- CHANGELOG.rst | 2 +- setup.py | 2 +- src/isal/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a248d5ee..86dc3447 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. -version 1.6.0-dev +version 1.6.0 ----------------- + Fix a bug where compression levels for IGzipFile where checked in read mode. + Update statically linked ISA-L release to 2.31.0 diff --git a/setup.py b/setup.py index 008e16ba..2212f8c2 100644 --- a/setup.py +++ b/setup.py @@ -130,7 +130,7 @@ def build_isa_l(): setup( name="isal", - version="1.6.0-dev", + version="1.6.0", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index d6c02fdb..2899e7df 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -16,4 +16,4 @@ "__version__" ] -__version__ = "1.6.0-dev" +__version__ = "1.6.0" From 634c77586fc123c6c7c30da1a135dc791edb2e5b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 19 Feb 2024 10:01:35 +0100 Subject: [PATCH 036/118] Increment version number --- setup.py | 2 +- src/isal/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2212f8c2..d5ec70f3 100644 --- a/setup.py +++ b/setup.py @@ -130,7 +130,7 @@ def build_isa_l(): setup( name="isal", - version="1.6.0", + version="1.7.0-dev", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index 2899e7df..33d577b5 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -16,4 +16,4 @@ "__version__" ] -__version__ = "1.6.0" +__version__ = "1.7.0-dev" From 525c20c71eb15e7e505a2a03048af020909f6ba1 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 11 Mar 2024 11:35:47 +0100 Subject: [PATCH 037/118] igzip_threaded.open no longer closes streams that are passed to it --- CHANGELOG.rst | 4 ++++ src/isal/igzip_threaded.py | 14 +++++++++----- tests/test_igzip_threaded.py | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 86dc3447..4155e5d2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 1.6.1 +----------------- ++ Fix a bug where streams that were passed to igzip_threaded.open where closed. + version 1.6.0 ----------------- + Fix a bug where compression levels for IGzipFile where checked in read mode. diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index adb2c24d..cd8b4238 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -78,16 +78,18 @@ def open(filename, mode="rb", compresslevel=igzip._COMPRESS_LEVEL_TRADEOFF, def open_as_binary_stream(filename, open_mode): if isinstance(filename, (str, bytes)) or hasattr(filename, "__fspath__"): binary_file = builtins.open(filename, open_mode) + closefd = True elif hasattr(filename, "read") or hasattr(filename, "write"): binary_file = filename + closefd = False else: raise TypeError("filename must be a str or bytes object, or a file") - return binary_file + return binary_file, closefd class _ThreadedGzipReader(io.RawIOBase): def __init__(self, filename, queue_size=2, block_size=1024 * 1024): - self.raw = open_as_binary_stream(filename, "rb") + self.raw, self.closefd = open_as_binary_stream(filename, "rb") self.fileobj = igzip._IGzipReader(self.raw, buffersize=8 * block_size) self.pos = 0 self.read_file = False @@ -155,7 +157,8 @@ def close(self) -> None: self.running = False self.worker.join() self.fileobj.close() - self.raw.close() + if self.closefd: + self.raw.close() self._closed = True @property @@ -246,7 +249,7 @@ def __init__(self, self._crc = 0 self.running = False self._size = 0 - self.raw = open_as_binary_stream(filename, mode) + self.raw, self.closefd = open_as_binary_stream(filename, mode) self._closed = False self._write_gzip_header() self.start() @@ -329,7 +332,8 @@ def close(self) -> None: trailer = struct.pack(" Date: Mon, 11 Mar 2024 13:36:27 +0100 Subject: [PATCH 038/118] Set stable version --- setup.py | 2 +- src/isal/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d5ec70f3..447a78c5 100644 --- a/setup.py +++ b/setup.py @@ -130,7 +130,7 @@ def build_isa_l(): setup( name="isal", - version="1.7.0-dev", + version="1.6.1", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index 33d577b5..bf77db23 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -16,4 +16,4 @@ "__version__" ] -__version__ = "1.7.0-dev" +__version__ = "1.6.1" From db1d6fe943fe240b19add9b0734b522504e7f3d5 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 11 Mar 2024 14:06:25 +0100 Subject: [PATCH 039/118] Increment version --- setup.py | 2 +- src/isal/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 447a78c5..d5ec70f3 100644 --- a/setup.py +++ b/setup.py @@ -130,7 +130,7 @@ def build_isa_l(): setup( name="isal", - version="1.6.1", + version="1.7.0-dev", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index bf77db23..33d577b5 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -16,4 +16,4 @@ "__version__" ] -__version__ = "1.6.1" +__version__ = "1.7.0-dev" From 748c07efe0d99558d20d381cc4b4947de6a27fa4 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 27 Mar 2024 17:19:04 +0100 Subject: [PATCH 040/118] Properly place differences anchor --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fc3d5195..9213fe3f 100644 --- a/README.rst +++ b/README.rst @@ -101,8 +101,6 @@ Installation is supported on Linux, Windows and MacOS. For more advanced installation options check the `documentation `_. -.. _differences-with-zlib-and-gzip-modules: - python-isal as a dependency in your project ------------------------------------------- @@ -125,6 +123,8 @@ your project please list a python-isal dependency as follows. .. dependency end +.. _differences-with-zlib-and-gzip-modules: + Differences with zlib and gzip modules -------------------------------------- From ea0a04c94ea6105200f3daa9c38a57c45efffbf2 Mon Sep 17 00:00:00 2001 From: cielavenir Date: Thu, 28 Mar 2024 16:38:28 +0900 Subject: [PATCH 041/118] update isa-l --- src/isal/isa-l | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isal/isa-l b/src/isal/isa-l index bd226375..7b30857e 160000 --- a/src/isal/isa-l +++ b/src/isal/isa-l @@ -1 +1 @@ -Subproject commit bd226375027899087bd48f3e59b910430615cc0a +Subproject commit 7b30857e20b84e5afab1a28291189b9dc571110d From 154334695e3ef488917e9c25c1b1ccd5e5fb8c41 Mon Sep 17 00:00:00 2001 From: cielavenir Date: Thu, 28 Mar 2024 16:40:23 +0900 Subject: [PATCH 042/118] add macos-14 CI --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5193c14a..450e86f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,9 @@ jobs: - "pypy-3.10" os: ["ubuntu-latest"] include: - - os: "macos-latest" + - os: "macos-13" + python-version: 3.8 + - os: "macos-14" python-version: 3.8 - os: "windows-latest" python-version: 3.8 @@ -132,7 +134,7 @@ jobs: shell: bash -l {0} strategy: matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest", "macos-13", "macos-14", "windows-latest"] python_version: [ "python" ] include: - os: "ubuntu-latest" From b278f621a23afae747ef883103da69ba04c0fbd0 Mon Sep 17 00:00:00 2001 From: cielavenir Date: Thu, 28 Mar 2024 16:40:44 +0900 Subject: [PATCH 043/118] add CI branch name --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 450e86f1..4dd4dcc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,7 @@ on: branches: - develop - main + - macosCI tags: - "*" From 81437af237b61cad165c0d66f07e579189dc3aa9 Mon Sep 17 00:00:00 2001 From: cielavenir Date: Thu, 28 Mar 2024 16:44:32 +0900 Subject: [PATCH 044/118] arm mac requires 3.10 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dd4dcc9..6de212e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,11 +69,11 @@ jobs: os: ["ubuntu-latest"] include: - os: "macos-13" - python-version: 3.8 + python-version: "3.8" - os: "macos-14" - python-version: 3.8 + python-version: "3.10" - os: "windows-latest" - python-version: 3.8 + python-version: "3.8" steps: - uses: actions/checkout@v2.3.4 with: From 06842c437a4a3381744c53ca132a4c94d29272d7 Mon Sep 17 00:00:00 2001 From: cielavenir Date: Thu, 28 Mar 2024 16:48:41 +0900 Subject: [PATCH 045/118] revert miniconda change --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6de212e2..f04ae5a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: shell: bash -l {0} strategy: matrix: - os: ["ubuntu-latest", "macos-13", "macos-14", "windows-latest"] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] python_version: [ "python" ] include: - os: "ubuntu-latest" From b9b4598a1304c0133a451570ce1da972d2ab1b33 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 29 Mar 2024 11:09:53 +0100 Subject: [PATCH 046/118] Update ci.yml --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f04ae5a0..9dc03cc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,6 @@ on: branches: - develop - main - - macosCI tags: - "*" From cba0c33389811c1117b2fa7b3fa39616c2e9b450 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 29 Mar 2024 11:14:02 +0100 Subject: [PATCH 047/118] Add macos-14 to the build roster --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dc03cc0..bfdaf859 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,7 +170,8 @@ jobs: matrix: os: - ubuntu-latest - - macos-latest + - macos-13 + - macos-14 - windows-latest cibw_archs_linux: ["x86_64"] cibw_before_all_linux: From 244104405947a5e41f16926884bf84ac656de105 Mon Sep 17 00:00:00 2001 From: Jan Glaubitz Date: Fri, 3 May 2024 09:55:00 +0200 Subject: [PATCH 048/118] Fix setup.py for FreeBSD, NetBSD --- setup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d5ec70f3..e7570859 100644 --- a/setup.py +++ b/setup.py @@ -20,8 +20,11 @@ ISA_L_SOURCE = os.path.join("src", "isal", "isa-l") +SYSTEM_IS_BSD = (sys.platform.startswith("freebsd") or + sys.platform.startswith("netbsd")) SYSTEM_IS_UNIX = (sys.platform.startswith("linux") or - sys.platform.startswith("darwin")) + sys.platform.startswith("darwin") or + SYSTEM_IS_BSD) SYSTEM_IS_WINDOWS = sys.platform.startswith("win") # Since pip builds in a temp directory by default, setting a fixed file in @@ -113,7 +116,10 @@ def build_isa_l(): cflags_param = "CFLAGS_aarch64" else: cflags_param = "CFLAGS_" - subprocess.run(["make", "-j", str(cpu_count), "-f", "Makefile.unx", + make_cmd = "make" + if SYSTEM_IS_BSD: + make_cmd = "gmake" + subprocess.run([make_cmd, "-j", str(cpu_count), "-f", "Makefile.unx", "isa-l.h", "bin/isa-l.a", f"{cflags_param}={build_env.get('CFLAGS', '')}"], **run_args) From 9349a7e558eb565c2d482560c56529cfdaf93f2f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 3 May 2024 16:26:15 +0200 Subject: [PATCH 049/118] Update miniconda setup version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5193c14a..ed9f1a8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,7 +142,7 @@ jobs: with: submodules: recursive - name: Install miniconda. - uses: conda-incubator/setup-miniconda@v2.0.1 # https://github.com/conda-incubator/setup-miniconda. + uses: conda-incubator/setup-miniconda@v3 # https://github.com/conda-incubator/setup-miniconda. with: channels: conda-forge,defaults - name: Install requirements (universal) From 4a3a1199750ca5849467881f0a8de35551140806 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 6 May 2024 10:40:09 +0200 Subject: [PATCH 050/118] Only test dynamic install on x86 macos --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed9f1a8b..17292cf8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,7 +132,7 @@ jobs: shell: bash -l {0} strategy: matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest", "macos-13", "windows-latest"] python_version: [ "python" ] include: - os: "ubuntu-latest" From ec6f1ab7e17316fc6fe8e1d422bd7d6dbcf03d7d Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 15 Jul 2024 10:12:31 +0200 Subject: [PATCH 051/118] Inherit READ, WRITE values from gzip library --- CHANGELOG.rst | 5 +++++ src/isal/igzip.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4155e5d2..90e0cfda 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 1.7.0-dev +----------------- ++ Fix a bug where READ and WRITE in isal.igzip were inconsistent with the + values in gzip on Python 3.13 + version 1.6.1 ----------------- + Fix a bug where streams that were passed to igzip_threaded.open where closed. diff --git a/src/isal/igzip.py b/src/isal/igzip.py index 3c2ed2cb..4cab8e0f 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -55,7 +55,8 @@ READ_BUFFER_SIZE = 512 * 1024 FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 -READ, WRITE = 1, 2 +READ = gzip.READ +WRITE = gzip.WRITE BadGzipFile = gzip.BadGzipFile # type: ignore From df7f5081ded1f189f1e544f9360491078e82a57b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 15 Jul 2024 10:00:20 +0200 Subject: [PATCH 052/118] Use a patched version of ISA-L 2.31 for MacOS arm64 --- .gitmodules | 2 +- CHANGELOG.rst | 2 ++ src/isal/isa-l | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index d79a05a7..11a84d8b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "src/isal/isa-l"] path = src/isal/isa-l - url = https://github.com/intel/isa-l.git + url = https://github.com/rhpvorderman/isa-l.git diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 90e0cfda..c6c875a6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,8 @@ Changelog version 1.7.0-dev ----------------- ++ Include a patched ISA-L version 2.31. The applied patches make compilation + and wheelbuilding on MacOS ARM64 possible. + Fix a bug where READ and WRITE in isal.igzip were inconsistent with the values in gzip on Python 3.13 diff --git a/src/isal/isa-l b/src/isal/isa-l index 7b30857e..e3569fb7 160000 --- a/src/isal/isa-l +++ b/src/isal/isa-l @@ -1 +1 @@ -Subproject commit 7b30857e20b84e5afab1a28291189b9dc571110d +Subproject commit e3569fb7224f071fe5ae510453f4451497385e44 From 216045644a82eff623c6e17f8d3e1f1de22fc9f2 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 11:18:16 +0200 Subject: [PATCH 053/118] Add a development chapter --- README.rst | 18 ++++++++++++++++++ docs/index.rst | 7 +++++++ 2 files changed, 25 insertions(+) diff --git a/README.rst b/README.rst index 9213fe3f..9b6961cc 100644 --- a/README.rst +++ b/README.rst @@ -166,6 +166,24 @@ are also very welcome. Please report them on the `github issue tracker .. contributing end +Development +----------- +.. development start + +The repository needs to be cloned recursively to make sure the +`ISA-L `_ repository is checked out: +``git clone --recursive https://github.com/pycompression/python-isal.git``. If +the repository is already checked out you can use ``git submodule update --init``. + +Patches should be made on a feature branch. To run the testing install ``tox`` +with ``pip install tox`` and run the commands ``tox -e lint`` and +``tox``. That will run most of the testing that is also performed by the CI. +For changes to the documentation run ``tox -e docs``. For changes to the C +code please also run ``tox -e asan`` to check for memory leaks. This requires +libasan to be installed. + +.. development end + Acknowledgements ---------------- diff --git a/docs/index.rst b/docs/index.rst index 2d540938..f06feb9a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -158,6 +158,13 @@ Contributing :start-after: .. contributing start :end-before: .. contributing end +=========== +Development +=========== +.. include:: includes/README.rst + :start-after: .. development start + :end-before: .. development end + ================ Acknowledgements ================ From 48a366c30053ea9d1b3264b4427a6467682ecafe Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 11:21:23 +0200 Subject: [PATCH 054/118] Always run documentation tests --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e239f990..07d211c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,6 @@ name: Continous integration on: pull_request: - paths-ignore: - - 'docs/**' - - '*.rst' push: branches: - develop From 19b6bae08f95944acbeb1ff59fff052a05ddaeba Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 11:33:20 +0200 Subject: [PATCH 055/118] Temporarily pin sphinx-argparse version --- requirements-docs.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 051c278d..39850f6b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,4 +1,5 @@ sphinx setuptools sphinx-rtd-theme -sphinx-argparse \ No newline at end of file +# See https://github.com/sphinx-doc/sphinx-argparse/issues/56 +sphinx-argparse <0.5.0 \ No newline at end of file From 726ceed3361ae92d1d4dbf4dcb5a8e12ba9f0557 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 11:55:05 +0200 Subject: [PATCH 056/118] Use version from the package --- docs/conf.py | 13 +++---------- requirements-docs.txt | 1 - 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1f6b7297..f92ec8a2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,27 +4,20 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -from distutils.dist import DistributionMetadata -from pathlib import Path - -import pkg_resources +from isal import __version__ # -- Project information ----------------------------------------------------- # Get package information from the installed package. -package = pkg_resources.get_distribution("isal") -metadata_file = Path(package.egg_info) / Path(package.PKG_INFO) -metadata = DistributionMetadata(path=str(metadata_file)) project = 'python-isal' copyright = '2020, Leiden University Medical Center' author = 'Leiden University Medical Center' # The short X.Y version -version = package.parsed_version.base_version +version = __version__ # The full version, including alpha/beta/rc tags -release = package.version - +release = __version__ # -- General configuration --------------------------------------------------- diff --git a/requirements-docs.txt b/requirements-docs.txt index 39850f6b..89dc59f1 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,5 +1,4 @@ sphinx -setuptools sphinx-rtd-theme # See https://github.com/sphinx-doc/sphinx-argparse/issues/56 sphinx-argparse <0.5.0 \ No newline at end of file From 7fc677cccf7f5efbea3fcc0dbdfe27461582e40e Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 12:03:34 +0200 Subject: [PATCH 057/118] Add a link to the ISA-L build requirementes --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 9b6961cc..734b2f74 100644 --- a/README.rst +++ b/README.rst @@ -182,6 +182,10 @@ For changes to the documentation run ``tox -e docs``. For changes to the C code please also run ``tox -e asan`` to check for memory leaks. This requires libasan to be installed. +Building requires the +`ISA-L build requirements `_ +as well. + .. development end Acknowledgements From dacbe52fbb9fe9f230eb98f1f1b5623f03bf64e9 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 14:53:40 +0200 Subject: [PATCH 058/118] Build version with versioningit --- .gitignore | 2 ++ pyproject.toml | 8 +++++++- setup.py | 4 +++- src/isal/__init__.py | 3 +-- src/isal/_version.pyi | 8 ++++++++ 5 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 src/isal/_version.pyi diff --git a/.gitignore b/.gitignore index b6e47617..8e29d4aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +src/isal/_version.py + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/pyproject.toml b/pyproject.toml index 62f5f82c..b98fe918 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,9 @@ [build-system] -requires = ["setuptools>=51", "wheel"] +requires = ["setuptools>=64", "versioningit>=1.1.0"] build-backend = "setuptools.build_meta" + +[tool.versioningit.vcs] +method="git" + +[tool.versioningit.write] +file = "src/isal/_version.py" diff --git a/setup.py b/setup.py index e7570859..84a92cf7 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,8 @@ from setuptools import Extension, find_packages, setup from setuptools.command.build_ext import build_ext +import versioningit + ISA_L_SOURCE = os.path.join("src", "isal", "isa-l") SYSTEM_IS_BSD = (sys.platform.startswith("freebsd") or @@ -136,7 +138,7 @@ def build_isa_l(): setup( name="isal", - version="1.7.0-dev", + version=versioningit.get_version(), description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index 33d577b5..7addbeef 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -7,6 +7,7 @@ from ._isal import (ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION, ISAL_VERSION) +from ._version import __version__ __all__ = [ "ISAL_MAJOR_VERSION", @@ -15,5 +16,3 @@ "ISAL_VERSION", "__version__" ] - -__version__ = "1.7.0-dev" diff --git a/src/isal/_version.pyi b/src/isal/_version.pyi new file mode 100644 index 00000000..bd8c70e0 --- /dev/null +++ b/src/isal/_version.pyi @@ -0,0 +1,8 @@ +# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +# 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 +# Python Software Foundation; All Rights Reserved + +# This file is part of python-isal which is distributed under the +# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. + +__version__: int From 268d50b4fd345897fc19be729798640f6a44bb16 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 15:03:03 +0200 Subject: [PATCH 059/118] Update MANIFEST.in --- MANIFEST.in | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 3ef9dc16..8cc2b029 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,8 @@ graft src/isal/isa-l include src/isal/*.h +prune tests +prune docs +prune benchmark_scripts +exclude requirements-docs.txt +exclude codecov.yml +exclude .readthedocs.yml From 371fcd9bd12a7425250ec43f1dc3bc43b4e3d098 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 15:05:33 +0200 Subject: [PATCH 060/118] Update release procedure to reflect automated versioning changes --- .github/release_checklist.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/release_checklist.md b/.github/release_checklist.md index 7093fe46..a38d8661 100644 --- a/.github/release_checklist.md +++ b/.github/release_checklist.md @@ -2,9 +2,7 @@ Release checklist - [ ] Check outstanding issues on JIRA and Github. - [ ] Check [latest documentation](https://python-isal.readthedocs.io/en/latest/) looks fine. - [ ] Create a release branch. - - [ ] Set version to a stable number. - [ ] Change current development version in `CHANGELOG.rst` to stable version. - - [ ] Change the version in `__init__.py` - [ ] Check if the address sanitizer does not find any problems using `tox -e asan` - [ ] Merge the release branch into `main`. - [ ] Created an annotated tag with the stable version number. Include changes From 33f51263e88d32f8ec4c614238e00a4307916f02 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 15:16:42 +0200 Subject: [PATCH 061/118] Use a clearly wrong default tag to notify when something is amiss --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b98fe918..bf5fe99c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ build-backend = "setuptools.build_meta" [tool.versioningit.vcs] method="git" +default-tag = "v0.0.0" [tool.versioningit.write] file = "src/isal/_version.py" From cf11a8119bed2faa0bc51d044b464b955e152755 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 15:33:54 +0200 Subject: [PATCH 062/118] Use the latest version of checkout and fully clone for deploy builds --- .github/workflows/ci.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07d211c4..98285286 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python 3.8 @@ -33,7 +33,7 @@ jobs: - twine_check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python 3.8 @@ -71,7 +71,7 @@ jobs: - os: "windows-latest" python-version: "3.8" steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} @@ -107,7 +107,7 @@ jobs: python_version: - "3.8" steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - uses: uraimo/run-on-arch-action@v2.5.0 @@ -137,7 +137,7 @@ jobs: - os: "ubuntu-latest" python_version: "pypy" steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - name: Install miniconda. @@ -186,9 +186,10 @@ jobs: cibw_archs_linux: "aarch64" cibw_before_all_linux: "true" # The true command exits with 0 steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive + fetch-depth: 0 # Fetch everything to get accurately versioned tag. - uses: actions/setup-python@v2 name: Install Python - name: Install cibuildwheel twine wheel From 442967e56fa43d87b31a301e0ed843794f76e561 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 17 Jul 2024 08:53:43 +0200 Subject: [PATCH 063/118] Fix aarch64 build --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98285286..30edca83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,11 @@ jobs: arch: none distro: none base_image: "--platform=linux/arm64 quay.io/pypa/manylinux2014_aarch64" + # versioningit needs an accessible git repository but the container + # is run as root, which is different from the repository user. + # use git config to override this. run: |- + git config --global --add safe.directory $PWD CFLAGS="-DNDEBUG -g0" python${{matrix.python_version}} -m pip install . pytest python${{matrix.python_version}} -m pytest tests From 17829b621743cb552a5ae63fad4525efd27e3479 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 19 Jul 2024 06:41:38 +0200 Subject: [PATCH 064/118] Create header inline rather than calling a function --- src/isal/igzip.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/isal/igzip.py b/src/isal/igzip.py index 4cab8e0f..8b5c0bf3 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -225,13 +225,10 @@ def write(self, data): _IGzipReader = _GzipReader -def _create_simple_gzip_header(compresslevel: int, - mtime: Optional[SupportsInt] = None) -> bytes: - """ - Write a simple gzip header with no extra fields. - :param compresslevel: Compresslevel used to determine the xfl bytes. - :param mtime: The mtime (must support conversion to a 32-bit integer). - :return: A bytes object representing the gzip header. +def compress(data, compresslevel: int = _COMPRESS_LEVEL_BEST, *, + mtime: Optional[SupportsInt] = None) -> bytes: + """Compress data in one shot and return the compressed string. + Optional argument is the compression level, in range of 0-3. """ if mtime is None: mtime = time.time() @@ -240,14 +237,7 @@ def _create_simple_gzip_header(compresslevel: int, xfl = 4 if compresslevel == _COMPRESS_LEVEL_FAST else 0 # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra # fields added to header), mtime, xfl and os (255 for unknown OS). - return struct.pack(" Date: Fri, 19 Jul 2024 06:48:39 +0200 Subject: [PATCH 065/118] Use shutil.copyfileobj to do file object copying --- src/isal/igzip.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/isal/igzip.py b/src/isal/igzip.py index 8b5c0bf3..18a8c5cb 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -33,6 +33,7 @@ import gzip import io import os +import shutil import struct import sys import time @@ -358,11 +359,7 @@ def main(): out_file = sys.stdout.buffer try: - while True: - block = in_file.read(args.buffer_size) - if block == b"": - break - out_file.write(block) + shutil.copyfileobj(in_file, out_file, args.buffer_size) finally: if in_file is not sys.stdin.buffer: in_file.close() From 803dbebf24ca6af8bd203458a11d4d9b53e3d605 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 19 Jul 2024 09:17:13 +0200 Subject: [PATCH 066/118] Update the changes in the files modified from CPython --- src/isal/igzip.py | 6 +----- src/isal/isal_zlibmodule.c | 5 ++--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/isal/igzip.py b/src/isal/igzip.py index 18a8c5cb..af1b4712 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -9,11 +9,7 @@ # Changes compared to CPython: # - Subclassed GzipFile to IGzipFile. Methods that included calls to zlib have # been overwritten with the same methods, but now calling to isal_zlib. -# - _GzipReader uses a igzip_lib.IgzipDecompressor. This Decompressor is -# derived from the BZ2Decompressor as such it does not produce an unconsumed -# tail but keeps the read data internally. This prevents unnecessary copying -# of data. To accomodate this, the read method has been rewritten. -# - _GzipReader._add_read_data uses isal_zlib.crc32 instead of zlib.crc32. +# - _GzipReader is implemented in C in isal_zlib and allows dropping the GIL. # - Gzip.compress does not use a GzipFile to compress in memory, but creates a # simple header using _create_simple_gzip_header and compresses the data with # igzip_lib.compress using the DECOMP_GZIP_NO_HDR flag. This change was diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 4ba540ec..5463e22f 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -14,14 +14,13 @@ Changes compared to CPython: - Zlib to ISA-L conversion functions were included. - All compression and checksum functions from zlib replaced with ISA-L compatible functions. -- No locks in Compress and Decompress objects. These were deemed unnecessary - as the ISA-L functions do not allocate memory, unlike the zlib - counterparts. - zlib.compress also has a 'wbits' argument. This change was included in Python 3.11. It allows for faster gzip compression by using isal_zlib.compress(data, wbits=31). - Argument parsers were written using th CPython API rather than argument clinic. +- Created a GzipReader class that implements gzip reading in C, reducing a lot + of overhead compared to the gzip.py:_GzipReader class. */ #include "isal_shared.h" From 7667e6fe42a70a6407ffe66be86bca4a2b8a82f3 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 19 Jul 2024 09:20:59 +0200 Subject: [PATCH 067/118] Update changelog with simplification changes --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c6c875a6..c931f659 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,8 @@ version 1.7.0-dev and wheelbuilding on MacOS ARM64 possible. + Fix a bug where READ and WRITE in isal.igzip were inconsistent with the values in gzip on Python 3.13 ++ Small simplifications to the ``igzip.compress`` function, which should lead + to less overhead. version 1.6.1 ----------------- From 7926c93af582c9cc532215a66788322ca59bc1f3 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 9 Aug 2024 14:14:29 +0200 Subject: [PATCH 068/118] Prepare release 1.7.0 --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c931f659..c7f686ac 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. -version 1.7.0-dev +version 1.7.0 ----------------- + Include a patched ISA-L version 2.31. The applied patches make compilation and wheelbuilding on MacOS ARM64 possible. From c8b47e24287adc58078c99d764f13bb3411eecf4 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 9 Aug 2024 16:05:25 +0200 Subject: [PATCH 069/118] Do not deploy to problematic MacOS arm64 builds --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30edca83..3e767f53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,7 +215,7 @@ jobs: - name: Build wheels run: cibuildwheel --output-dir dist env: - CIBW_SKIP: "*-win32 *-manylinux_i686" # Skip 32 bit + CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_*arm64 cp39-macosx_*arm64" # Skip 32 bit and problematic mac builds. CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }} # Fully test the build wheels again. From 179f8cc02cd2a854e2deb95b9b98a8d7223a05f4 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 9 Aug 2024 16:10:12 +0200 Subject: [PATCH 070/118] Add 3.13 to the supported python versions --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 84a92cf7..7491870d 100644 --- a/setup.py +++ b/setup.py @@ -166,6 +166,7 @@ def build_isa_l(): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", From 2f2b749a7db43c92f4c0bf09f00615f0a4aeb48b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 9 Aug 2024 16:21:01 +0200 Subject: [PATCH 071/118] Skip existing uploads --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e767f53..f83ca894 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -264,13 +264,13 @@ jobs: # pypa/gh-action-pypi-publish@master does not work on OSX # Alpha, Beta and dev releases contain a - in the tag. if: contains(github.ref, '-') && startsWith(github.ref, 'refs/tags') - run: twine upload --repository-url https://test.pypi.org/legacy/ dist/* + run: twine upload --skip-existing --repository-url https://test.pypi.org/legacy/ dist/* env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} - name: Publish package to PyPI if: "!contains(github.ref, '-') && startsWith(github.ref, 'refs/tags')" - run: twine upload dist/* + run: twine upload --skip-existing dist/* env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} From 4229fc78390285552e63002cb63f53d0d573de0c Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 08:47:47 +0200 Subject: [PATCH 072/118] Fix documentation dependencies --- requirements-docs.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 89dc59f1..47081d94 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,4 +1,3 @@ sphinx sphinx-rtd-theme -# See https://github.com/sphinx-doc/sphinx-argparse/issues/56 -sphinx-argparse <0.5.0 \ No newline at end of file +sphinx-argparse From a1286cf546e76e5fb0709aa2a2741d15247aba7e Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 08:53:05 +0200 Subject: [PATCH 073/118] Reproduce blocking threads issue --- tests/test_igzip_threaded.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 336a5806..d211f81f 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -9,6 +9,8 @@ import io import itertools import os +import subprocess +import sys import tempfile from pathlib import Path @@ -218,3 +220,22 @@ def test_threaded_writer_does_not_close_stream(): assert not test_stream.closed test_stream.seek(0) assert gzip.decompress(test_stream.read()) == b"thisisatest" + + +@pytest.mark.timeout(5) +@pytest.mark.parametrize( + ["mode", "threads"], itertools.product(["rb", "wb"], [1, 2])) +def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): + program = tmp_path / "no_context_manager.py" + test_file = tmp_path / "output.gz" + # Write 40 mb input data to saturate read buffer. Because of the repetitive + # nature the resulting gzip file is very small (~40 KiB). + test_file.write_bytes(gzip.compress(b"test" * (10 * 1024 * 1024))) + with open(program, "wt") as f: + f.write("from isal import igzip_threaded\n") + f.write( + f"f = igzip_threaded.open('{test_file}', " + f"mode='{mode}', threads={threads})\n" + ) + f.write("raise Exception('Error')\n") + subprocess.run([sys.executable, str(program)]) From a0639b9e44e38575c4dc177597e0fbe7e8f6d6c0 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 09:00:58 +0200 Subject: [PATCH 074/118] Prevent threads from blocking python exit --- CHANGELOG.rst | 5 +++++ src/isal/igzip_threaded.py | 12 +++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c7f686ac..fe22d86e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 1.7.1-dev +----------------- ++ Prevent threaded opening from blocking python exit when an error is thrown + in the calling thread. + version 1.7.0 ----------------- + Include a patched ISA-L version 2.31. The applied patches make compilation diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index cd8b4238..da076e9b 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -101,6 +101,7 @@ def __init__(self, filename, queue_size=2, block_size=1024 * 1024): self.worker = threading.Thread(target=self._decompress) self._closed = False self.running = True + self._calling_thread = threading.current_thread() self.worker.start() def _check_closed(self, msg=None): @@ -110,7 +111,7 @@ def _check_closed(self, msg=None): def _decompress(self): block_size = self.block_size block_queue = self.queue - while self.running: + while self.running and self._calling_thread.is_alive(): try: data = self.fileobj.read(block_size) except Exception as e: @@ -118,7 +119,7 @@ def _decompress(self): return if not data: return - while self.running: + while self.running and self._calling_thread.is_alive(): try: block_queue.put(data, timeout=0.05) break @@ -215,6 +216,7 @@ def __init__(self, if "b" not in mode: mode += "b" self.lock = threading.Lock() + self._calling_thread = threading.current_thread() self.exception: Optional[Exception] = None self.level = level self.previous_block = b"" @@ -348,7 +350,7 @@ def _compress(self, index: int): try: data, zdict = in_queue.get(timeout=0.05) except queue.Empty: - if not self.running: + if not (self.running and self._calling_thread.is_alive()): return continue try: @@ -373,7 +375,7 @@ def _write(self): try: compressed, crc, data_length = output_queue.get(timeout=0.05) except queue.Empty: - if not self.running: + if not (self.running and self._calling_thread.is_alive()): self._crc = total_crc self._size = size return @@ -396,7 +398,7 @@ def _compress_and_write(self): try: data, zdict = in_queue.get(timeout=0.05) except queue.Empty: - if not self.running: + if not (self.running and self._calling_thread.is_alive()): self._crc = total_crc self._size = size return From ea30d6f93f3dde7c773b1807b2444a845e641bcf Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 09:13:48 +0200 Subject: [PATCH 075/118] Reproduce faulty flushing behavior --- tests/test_igzip_threaded.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index d211f81f..41c61bfe 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -239,3 +239,19 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): ) f.write("raise Exception('Error')\n") subprocess.run([sys.executable, str(program)]) + + +@pytest.mark.parametrize("threads", [1, 2]) +def test_flush(tmp_path, threads): + test_file = tmp_path / "output.gz" + with igzip_threaded.open(test_file, "wb", threads=threads) as f: + f.write(b"1") + f.flush() + assert gzip.decompress(test_file.read_bytes()) == b"1" + f.write(b"2") + f.flush() + assert gzip.decompress(test_file.read_bytes()) == b"12" + f.write(b"3") + f.flush() + assert gzip.decompress(test_file.read_bytes()) == b"123" + assert gzip.decompress(test_file.read_bytes()) == b"123" From a670f984970c52f47babc8ba656311f81a150ea5 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 09:18:04 +0200 Subject: [PATCH 076/118] Make threaded writer streams flushable --- CHANGELOG.rst | 2 ++ src/isal/igzip_threaded.py | 49 +++++++++++++++++++------------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fe22d86e..cdc01206 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,8 @@ Changelog version 1.7.1-dev ----------------- ++ Fix a bug where flushing files when writing in threaded mode did not work + properly. + Prevent threaded opening from blocking python exit when an error is thrown in the calling thread. diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index da076e9b..7f1c94fc 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -60,7 +60,7 @@ def open(filename, mode="rb", compresslevel=igzip._COMPRESS_LEVEL_TRADEOFF, gzip_file = io.BufferedReader( _ThreadedGzipReader(filename, block_size=block_size)) else: - gzip_file = io.BufferedWriter( + gzip_file = FlushableBufferedWriter( _ThreadedGzipWriter( filename, mode.replace("t", "b"), @@ -167,6 +167,12 @@ def closed(self) -> bool: return self._closed +class FlushableBufferedWriter(io.BufferedWriter): + def flush(self): + super().flush() + self.raw.flush() + + class _ThreadedGzipWriter(io.RawIOBase): """ Write a gzip file using multiple threads. @@ -310,7 +316,7 @@ def write(self, b) -> int: self.input_queues[worker_index].put((data, zdict)) return len(data) - def flush(self): + def _end_gzip_stream(self): self._check_closed() # Wait for all data to be compressed for in_q in self.input_queues: @@ -318,22 +324,27 @@ def flush(self): # Wait for all data to be written for out_q in self.output_queues: out_q.join() + # Write an empty deflate block with a lost block marker. + self.raw.write(isal_zlib.compress(b"", wbits=-15)) + trailer = struct.pack(" None: if self._closed: return - self.flush() + self._end_gzip_stream() self.stop() if self.exception: self.raw.close() self._closed = True raise self.exception - # Write an empty deflate block with a lost block marker. - self.raw.write(isal_zlib.compress(b"", wbits=-15)) - trailer = struct.pack(" Date: Wed, 25 Sep 2024 09:24:44 +0200 Subject: [PATCH 077/118] Remove manual version update from the release checklist. --- .github/release_checklist.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/release_checklist.md b/.github/release_checklist.md index a38d8661..8ae899dc 100644 --- a/.github/release_checklist.md +++ b/.github/release_checklist.md @@ -9,7 +9,6 @@ Release checklist from CHANGELOG.rst. - [ ] Push tag to remote. This triggers the wheel/sdist build on github CI. - [ ] merge `main` branch back into `develop`. -- [ ] Add updated version number to develop. (`setup.py` and `src/isal/__init__.py`) - [ ] Build the new tag on readthedocs. Only build the last patch version of each minor version. So `1.1.1` and `1.2.0` but not `1.1.0`, `1.1.1` and `1.2.0`. - [ ] Create a new release on github. From 63f3bcd97b1d89fab36acb50a55bf76c1b62be8f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 09:27:50 +0200 Subject: [PATCH 078/118] Prepare release 1.7.1 --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cdc01206..2ffabcb8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. -version 1.7.1-dev +version 1.7.1 ----------------- + Fix a bug where flushing files when writing in threaded mode did not work properly. From dc92b7d1ca25e529400a01317e0db4771d3a05ca Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 09:38:37 +0200 Subject: [PATCH 079/118] Update upload-artifact version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f83ca894..ddfae28a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,7 +256,7 @@ jobs: run: | pip install build python -m build --sdist - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact with: name: "dist" path: "dist/" From 540ef864281c3cdd4bb9f6325a01442e68acad72 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 09:45:52 +0200 Subject: [PATCH 080/118] Update upload-artifact --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddfae28a..cb10819c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,7 +256,7 @@ jobs: run: | pip install build python -m build --sdist - - uses: actions/upload-artifact + - uses: actions/upload-artifact@v4 with: name: "dist" path: "dist/" From 463fdfe32131b2f762d9385e634649ed168a66d3 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 11:42:13 +0200 Subject: [PATCH 081/118] Make artifact path unique --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb10819c..ad7e3a14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -258,7 +258,7 @@ jobs: python -m build --sdist - uses: actions/upload-artifact@v4 with: - name: "dist" + name: "dist-${{ env.RUNNER_OS }}-${{ env.RUNNER-ARCH }}" path: "dist/" - name: Publish package to TestPyPI # pypa/gh-action-pypi-publish@master does not work on OSX From 16cdab746b940c6a3cee10cc0cdaf3c674133f24 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 11:54:23 +0200 Subject: [PATCH 082/118] Use correct environment variable --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad7e3a14..c3d1c553 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -258,7 +258,7 @@ jobs: python -m build --sdist - uses: actions/upload-artifact@v4 with: - name: "dist-${{ env.RUNNER_OS }}-${{ env.RUNNER-ARCH }}" + name: "dist-${{ runner.os }}-${{ runner.arch }}" path: "dist/" - name: Publish package to TestPyPI # pypa/gh-action-pypi-publish@master does not work on OSX From dafecccf7474c2405a9c8c08c448ff5e9040f606 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 13:22:09 +0200 Subject: [PATCH 083/118] Add another source of uniqueness --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3d1c553..778f4cba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -258,7 +258,7 @@ jobs: python -m build --sdist - uses: actions/upload-artifact@v4 with: - name: "dist-${{ runner.os }}-${{ runner.arch }}" + name: "dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cibw_archs_linux }}" path: "dist/" - name: Publish package to TestPyPI # pypa/gh-action-pypi-publish@master does not work on OSX From 414e76edd2968d26a6cf21ecd685047f0e154c0c Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Mon, 25 Nov 2024 00:06:37 +0100 Subject: [PATCH 084/118] __version__ is a string --- src/isal/_version.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isal/_version.pyi b/src/isal/_version.pyi index bd8c70e0..e59e7032 100644 --- a/src/isal/_version.pyi +++ b/src/isal/_version.pyi @@ -5,4 +5,4 @@ # This file is part of python-isal which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. -__version__: int +__version__: str From 29d36c4afa81eab4da12598124c6362addc67914 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 25 Nov 2024 09:05:54 +0100 Subject: [PATCH 085/118] Remove unused theme option from doc configuration --- docs/conf.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f92ec8a2..f2ec9dd2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,9 +42,6 @@ # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' -html_theme_options = dict( - display_version=True, -) # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, From 9f776aa6d2749232846a7e4a83d4061c7803350d Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 25 Nov 2024 09:16:37 +0100 Subject: [PATCH 086/118] Fix length typo --- src/isal/isal_zlibmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 5463e22f..48bf6861 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -266,7 +266,7 @@ PyDoc_STRVAR(isal_zlib_crc32_combine__doc__, " crc2\n" " the second crc32 checksum\n" " crc2_length\n" -" the lenght of the data block crc2 was calculated from\n" +" the length of the data block crc2 was calculated from\n" ); From 51163436a46153ad6015591e8d208dae0d63f3f7 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 8 Jan 2025 09:16:42 +0100 Subject: [PATCH 087/118] Update internal isa-l version to 2.31.1 --- .gitmodules | 2 +- CHANGELOG.rst | 6 ++++++ src/isal/isa-l | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 11a84d8b..d79a05a7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "src/isal/isa-l"] path = src/isal/isa-l - url = https://github.com/rhpvorderman/isa-l.git + url = https://github.com/intel/isa-l.git diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2ffabcb8..f68a9cdc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. + +version develop +----------------- ++ Use upstream ISA-L version 2.31.1 which includes patches to make + installation possible. + version 1.7.1 ----------------- + Fix a bug where flushing files when writing in threaded mode did not work diff --git a/src/isal/isa-l b/src/isal/isa-l index e3569fb7..28305ade 160000 --- a/src/isal/isa-l +++ b/src/isal/isa-l @@ -1 +1 @@ -Subproject commit e3569fb7224f071fe5ae510453f4451497385e44 +Subproject commit 28305ade9e07213a2c602697a6a3a1e1eed2acca From 97607223a88a439d7636efc0e3cdf58886e33c27 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 10 Jan 2025 09:08:58 +0100 Subject: [PATCH 088/118] Use tagged version of 2.31.1 --- src/isal/isa-l | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isal/isa-l b/src/isal/isa-l index 28305ade..c387163f 160000 --- a/src/isal/isa-l +++ b/src/isal/isa-l @@ -1 +1 @@ -Subproject commit 28305ade9e07213a2c602697a6a3a1e1eed2acca +Subproject commit c387163fcbca62701d646149564c550c78a4f985 From 80bdfe59a1b16fc302e0e5a55df5a0a81e438b67 Mon Sep 17 00:00:00 2001 From: Mattias Ellert Date: Sat, 1 Mar 2025 05:43:08 +0100 Subject: [PATCH 089/118] uint16_t is 2 bytes wide - only read 2 bytes from mem --- src/isal/isal_zlibmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 48bf6861..67547be9 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1643,7 +1643,7 @@ static inline uint32_t load_u32_le(void *address) { static inline uint16_t load_u16_le(void *address) { #if PY_BIG_ENDIAN uint8_t *mem = address; - return mem[0] | (mem[1] << 8) | (mem[2] << 16) | (mem[3] << 24); + return mem[0] | (mem[1] << 8); #else return *(uint16_t *)address; #endif From 3bb00e48fa0c7d6af0018112cf3cd6aa65b0f7a3 Mon Sep 17 00:00:00 2001 From: Mattias Ellert Date: Sat, 1 Mar 2025 05:45:41 +0100 Subject: [PATCH 090/118] Reverse the order of copied bytes on big-endian Fixes test failures on s390x. --- CHANGELOG.rst | 2 ++ src/isal/isal_shared.h | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f68a9cdc..66921aa1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ version develop ----------------- + Use upstream ISA-L version 2.31.1 which includes patches to make installation possible. ++ Fix a bug where bytes were copied in the wrong order on big endian + architectures. Fixes test failures on s390x. version 1.7.1 ----------------- diff --git a/src/isal/isal_shared.h b/src/isal/isal_shared.h index 1a01f081..61610276 100644 --- a/src/isal/isal_shared.h +++ b/src/isal/isal_shared.h @@ -207,8 +207,15 @@ static int bitbuffer_copy(struct inflate_state *state, char *to, size_t n){ int remainder = bits_in_buffer % 8; // Shift the 8-byte bitbuffer read_in so that the bytes are aligned. uint64_t remaining_bytes = state->read_in >> remainder; + #if PY_BIG_ENDIAN + char *remaining_buffer = (char *)&remaining_bytes; + for (int i = 0; i < n; ++i) { + to[i] = remaining_buffer[7-i]; + } + #else // memcpy works because of little-endianness memcpy(to, &remaining_bytes, n); + #endif return 0; } From 44959b4c30367a923c0fa6937f9d81ddfe069ff4 Mon Sep 17 00:00:00 2001 From: Mattias Ellert Date: Sat, 1 Mar 2025 06:39:06 +0100 Subject: [PATCH 091/118] Enable building on GNU/Hurd platforms --- CHANGELOG.rst | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f68a9cdc..3de35e46 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,7 @@ version develop ----------------- + Use upstream ISA-L version 2.31.1 which includes patches to make installation possible. ++ Enable building on GNU/Hurd platforms. version 1.7.1 ----------------- diff --git a/setup.py b/setup.py index 7491870d..55c89496 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ sys.platform.startswith("netbsd")) SYSTEM_IS_UNIX = (sys.platform.startswith("linux") or sys.platform.startswith("darwin") or + sys.platform.startswith("gnu") or SYSTEM_IS_BSD) SYSTEM_IS_WINDOWS = sys.platform.startswith("win") From 55dd7941387896ee76943b31dd226a4e73475994 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 5 Mar 2025 09:27:31 +0100 Subject: [PATCH 092/118] Update actions to latest and greatest --- .github/workflows/ci.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 778f4cba..43544ce5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: with: submodules: recursive - name: Set up Python 3.8 - uses: actions/setup-python@v2.2.1 + uses: actions/setup-python@v5 with: python-version: 3.8 - name: Install tox @@ -37,7 +37,7 @@ jobs: with: submodules: recursive - name: Set up Python 3.8 - uses: actions/setup-python@v2.2.1 + uses: actions/setup-python@v5 with: python-version: 3.8 - name: Install isal @@ -75,7 +75,7 @@ jobs: with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2.2.1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install tox and upgrade setuptools @@ -89,10 +89,10 @@ jobs: run: brew install nasm if: runner.os == 'macOS' - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Install nasm (Windows) - uses: ilammy/setup-nasm@v1.2.0 + uses: ilammy/setup-nasm@v1 if: runner.os == 'Windows' - name: Run tests run: tox @@ -110,7 +110,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - uses: uraimo/run-on-arch-action@v2.5.0 + - uses: uraimo/run-on-arch-action@v3 name: Build & run test with: arch: none @@ -151,7 +151,7 @@ jobs: - name: Install requirements (universal) run: conda install isa-l ${{ matrix.python_version}} tox - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Run tests (dynamic link) run: tox @@ -194,7 +194,7 @@ jobs: with: submodules: recursive fetch-depth: 0 # Fetch everything to get accurately versioned tag. - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 name: Install Python - name: Install cibuildwheel twine wheel run: python -m pip install cibuildwheel twine wheel @@ -202,14 +202,14 @@ jobs: run: brew install nasm if: runner.os == 'macOS' - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Install nasm (Windows) - uses: ilammy/setup-nasm@v1.2.0 + uses: ilammy/setup-nasm@v1 if: runner.os == 'Windows' - name: Set up QEMU if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'aarch64'}} - uses: docker/setup-qemu-action@v1.0.1 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Build wheels From d08003ec2ce8f05d775eccd08c1cf5b344194546 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 5 Mar 2025 09:32:46 +0100 Subject: [PATCH 093/118] Test on released python 3.13 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43544ce5..b85762de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: - "3.10" - "3.11" - "3.12" - - "3.13-dev" + - "3.13" - "pypy-3.9" - "pypy-3.10" os: ["ubuntu-latest"] From 9af69cf31a7a8fa69f5d4bef4446812544b87cb9 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 5 Mar 2025 09:49:22 +0100 Subject: [PATCH 094/118] Set changelog to stable version --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cde1ed9b..805a953a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,10 +8,10 @@ Changelog .. that users understand how the changes affect the new version. -version develop +version 1.7.2 ----------------- + Use upstream ISA-L version 2.31.1 which includes patches to make - installation possible. + installation on MacOS ARM64 possible. + Fix a bug where bytes were copied in the wrong order on big endian architectures. Fixes test failures on s390x. + Enable building on GNU/Hurd platforms. From c4ea9e8ac193a291f8661e8669e6b28e9c11cdb4 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 5 Mar 2025 12:45:18 +0100 Subject: [PATCH 095/118] Use a low version for setup python for deployment --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b85762de..6b55ef76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,7 +194,7 @@ jobs: with: submodules: recursive fetch-depth: 0 # Fetch everything to get accurately versioned tag. - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v2 # Some issues where caused by higher versions. name: Install Python - name: Install cibuildwheel twine wheel run: python -m pip install cibuildwheel twine wheel From 5e320f523676c831cd3e99ae89d6e84e1fec86d4 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 5 Mar 2025 12:59:09 +0100 Subject: [PATCH 096/118] Use a lower version of sphinx for the time being --- requirements-docs.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 47081d94..8fb9ea6c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,3 +1,4 @@ -sphinx +# https://github.com/sphinx-doc/sphinx/issues/13415 +sphinx <8 sphinx-rtd-theme sphinx-argparse From 68989a34eab7143ec8ea317621d1fb4b1156688f Mon Sep 17 00:00:00 2001 From: Mattias Ellert Date: Fri, 14 Mar 2025 19:59:49 +0100 Subject: [PATCH 097/118] Add a dummy non-empty environment when using assert_python_{failure,ok} If the env_vars argument to assert_python_failure or assert_python_ok is empty the test will run in isolated mode (-I) which means that the PYTHONPATH environment variable will be ignored and the test fails because the isal module can not be found, or the test is run usung the system installed version of the module instead of the newly built module that should be tested. By adding a dummy entry to the env_vars argument the isolated mode is not used and the PYTHONPATH environment variable is not ignored and the test works as expected. --- CHANGELOG.rst | 4 ++++ tests/test_gzip_compliance.py | 26 +++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 805a953a..59757cb1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,10 @@ Changelog .. that users understand how the changes affect the new version. +version 1.7.3 +----------------- ++ Fix an issue where some tests failed because they ignored PYTHONPATH. + version 1.7.2 ----------------- + Use upstream ISA-L version 2.31.1 which includes patches to make diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index cc979317..bd11451d 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -845,9 +845,23 @@ def test_decompress_infile_outfile(self): self.assertTrue(os.path.exists(igzipname)) + # The following tests use assert_python_failure or assert_python_ok. + # + # If the env_vars argument to assert_python_failure or assert_python_ok + # is empty the test will run in isolated mode (-I) which means that the + # PYTHONPATH environment variable will be ignored and the test fails + # because the isal module can not be found, or the test is run usung the + # system installed version of the module instead of the newly built + # module that should be tested. + # + # By adding a dummy entry to the env_vars argument the isolated mode is + # not used and the PYTHONPATH environment variable is not ignored and + # the test works as expected. + def test_decompress_infile_outfile_error(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '-d', - 'thisisatest.out') + 'thisisatest.out', + **{'_dummy': '1'}) self.assertEqual(b"filename doesn't end in .gz: 'thisisatest.out'. " b"Cannot determine output filename.", err.strip()) @@ -872,7 +886,8 @@ def test_compress_infile_outfile_default(self): with open(local_testigzip, 'wb') as fp: fp.write(self.data) - rc, out, err = assert_python_ok('-m', 'isal.igzip', local_testigzip) + rc, out, err = assert_python_ok('-m', 'isal.igzip', local_testigzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(igzipname)) self.assertEqual(out, b'') @@ -891,7 +906,8 @@ def test_compress_infile_outfile(self): rc, out, err = assert_python_ok('-m', 'isal.igzip', compress_level, - local_testigzip) + local_testigzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(igzipname)) self.assertEqual(out, b'') @@ -901,7 +917,7 @@ def test_compress_infile_outfile(self): def test_compress_fast_best_are_exclusive(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '--fast', - '--best') + '--best', **{'_dummy': '1'}) self.assertIn( b"error: argument -3/--best: not allowed with argument -0/--fast", err) @@ -909,7 +925,7 @@ def test_compress_fast_best_are_exclusive(self): def test_decompress_cannot_have_flags_compression(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '--fast', - '-d') + '-d', **{'_dummy': '1'}) self.assertIn( b'error: argument -d/--decompress: not allowed with argument ' b'-0/--fast', From d08488ba37e068703ddff6168bc578d2dcdf8463 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 18 Mar 2025 11:32:50 +0100 Subject: [PATCH 098/118] Update changelog version --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 59757cb1..f9a053b7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,7 +8,7 @@ Changelog .. that users understand how the changes affect the new version. -version 1.7.3 +version 1.8.0-dev ----------------- + Fix an issue where some tests failed because they ignored PYTHONPATH. From 0215519b557f574352620a06255921d8025ecef1 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:08:28 +0200 Subject: [PATCH 099/118] Move to setuptools-scm --- MANIFEST.in | 2 ++ pyproject.toml | 62 +++++++++++++++++++++++++++++++++++++++++++++----- setup.cfg | 2 -- setup.py | 44 +---------------------------------- 4 files changed, 59 insertions(+), 51 deletions(-) delete mode 100644 setup.cfg diff --git a/MANIFEST.in b/MANIFEST.in index 8cc2b029..23ea5299 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,8 @@ graft src/isal/isa-l include src/isal/*.h prune tests prune docs +prune .github +exclude .git* prune benchmark_scripts exclude requirements-docs.txt exclude codecov.yml diff --git a/pyproject.toml b/pyproject.toml index bf5fe99c..425f0f9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,60 @@ [build-system] -requires = ["setuptools>=64", "versioningit>=1.1.0"] +requires = ["setuptools>=77", "setuptools-scm>=8"] build-backend = "setuptools.build_meta" -[tool.versioningit.vcs] -method="git" -default-tag = "v0.0.0" +[project] +name = "isal" +dynamic = ["version"] +description = """ +Faster zlib and gzip compatible compression and decompression by providing \ +python bindings for the ISA-L ibrary.""" +license="PSF-2.0" +keywords=["isal", "isa-l", "compression", "deflate", "gzip", "igzip"] +authors = [{name = "Leiden University Medical Center"}, + {email = "r.h.p.vorderman@lumc.nl"}] +readme = "README.rst" +requires-python = ">=3.8" # BadGzipFile imported +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: C", + "Development Status :: 5 - Production/Stable", + "Topic :: System :: Archiving :: Compression", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", +] +urls.homepage = "https://github.com/pycompression/python-isal" +urls.documentation = "python-isal.readthedocs.io" -[tool.versioningit.write] -file = "src/isal/_version.py" +[tool.setuptools_scm] +version_file = "src/isal/_version.py" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["isal"] + +[tool.setuptools.package-data] +isal = ['*.pyi', 'py.typed', 'isa-l/LICENSE', 'isa-l/README.md', 'isa-l/Release_notes.txt'] +[tool.setuptools.exclude-package-data] +isal = [ + "*.c", + "*.h", + "isa-l/*/*", + "isa-l/Mak*", + "isa-l/.*", + "isa-l/autogen.sh", + "isa-l/Doxyfile", + "isa-l/CONTRIBUTING.md", + "isa-l/SECURITY.md", + "isa-l/configure.ac", + "isa-l/isa-l.*", + "isa-l/libisal.pc.in", + "isa-l/make.inc", +] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 256134e1..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -license_files=LICENSE \ No newline at end of file diff --git a/setup.py b/setup.py index 55c89496..1ed64c2d 100644 --- a/setup.py +++ b/setup.py @@ -15,11 +15,9 @@ import tempfile from pathlib import Path -from setuptools import Extension, find_packages, setup +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext -import versioningit - ISA_L_SOURCE = os.path.join("src", "isal", "isa-l") SYSTEM_IS_BSD = (sys.platform.startswith("freebsd") or @@ -138,46 +136,6 @@ def build_isa_l(): setup( - name="isal", - version=versioningit.get_version(), - description="Faster zlib and gzip compatible compression and " - "decompression by providing python bindings for the ISA-L " - "library.", - author="Leiden University Medical Center", - author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now - long_description=Path("README.rst").read_text(), - long_description_content_type="text/x-rst", cmdclass={"build_ext": BuildIsalExt}, - license="PSF-2.0", - keywords="isal isa-l compression deflate gzip igzip threads", - zip_safe=False, - packages=find_packages('src'), - package_dir={'': 'src'}, - package_data={'isal': ['*.pyi', 'py.typed', - # Include isa-l LICENSE and other relevant files - # with the binary distribution. - 'isa-l/LICENSE', 'isa-l/README.md', - 'isa-l/Release_notes.txt']}, - url="https://github.com/pycompression/python-isal", - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: C", - "Development Status :: 5 - Production/Stable", - "Topic :: System :: Archiving :: Compression", - "License :: OSI Approved :: Python Software Foundation License", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - ], - python_requires=">=3.8", # BadGzipFile imported ext_modules=EXTENSIONS ) From 1da70a5bb2c4c3e7853064afe91b79098428bc35 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:08:44 +0200 Subject: [PATCH 100/118] Include tests in source distribution --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 23ea5299..4a48e873 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,5 @@ graft src/isal/isa-l include src/isal/*.h -prune tests prune docs prune .github exclude .git* From 26c2b45de6b9e9d540e924e8b8acbbb4fcd847b7 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:13:24 +0200 Subject: [PATCH 101/118] Fix CFLAGS on default behavior for later setuptools versions --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b55ef76..17dd7f83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,14 +243,14 @@ jobs: CIBW_ENVIRONMENT_LINUX: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=/tmp/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" CIBW_ENVIRONMENT_WINDOWS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}\build_cache CIBW_ENVIRONMENT_MACOS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" - name: Build sdist if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'x86_64'}} run: | From 736644860839a957fa6b47e2aea2f2727c7067ea Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:18:59 +0200 Subject: [PATCH 102/118] Update changelog with build changes --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f9a053b7..b79300d2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,10 @@ Changelog version 1.8.0-dev ----------------- ++ Change build backend to setuptools-scm which is more commonly used and + supported. ++ Include test packages in the source distribution, so source distribution + installations can be verified. + Fix an issue where some tests failed because they ignored PYTHONPATH. version 1.7.2 From fc7d5a56c5d8fb072e1b7a549ecaf1c339046622 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:31:35 +0200 Subject: [PATCH 103/118] Update supported python versions --- .github/workflows/ci.yml | 15 +++++++-------- CHANGELOG.rst | 1 + pyproject.toml | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17dd7f83..b886a730 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.9" - name: Install tox run: pip install tox - name: Lint @@ -36,10 +36,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.9" - name: Install isal run: sudo apt-get install libisal-dev - name: Install tox and upgrade setuptools and pip @@ -54,7 +54,6 @@ jobs: strategy: matrix: python-version: - - "3.8" - "3.9" - "3.10" - "3.11" @@ -65,11 +64,11 @@ jobs: os: ["ubuntu-latest"] include: - os: "macos-13" - python-version: "3.8" + python-version: "3.9" - os: "macos-14" python-version: "3.10" - os: "windows-latest" - python-version: "3.8" + python-version: "3.9" steps: - uses: actions/checkout@v4 with: @@ -105,7 +104,7 @@ jobs: strategy: matrix: python_version: - - "3.8" + - "3.9" steps: - uses: actions/checkout@v4 with: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b79300d2..c49acba7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,7 @@ Changelog version 1.8.0-dev ----------------- ++ Python 3.8 is no longer supported. + Change build backend to setuptools-scm which is more commonly used and supported. + Include test packages in the source distribution, so source distribution diff --git a/pyproject.toml b/pyproject.toml index 425f0f9d..8405984e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,14 +13,15 @@ keywords=["isal", "isa-l", "compression", "deflate", "gzip", "igzip"] authors = [{name = "Leiden University Medical Center"}, {email = "r.h.p.vorderman@lumc.nl"}] readme = "README.rst" -requires-python = ">=3.8" # BadGzipFile imported +requires-python = ">=3.9" # Because of setuptools version classifiers = [ "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", From b749782cf5253bc1dccb8cdaef32d26c2e96c6ac Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 28 May 2025 13:30:12 +0200 Subject: [PATCH 104/118] Mark extensions as free-threading-compatible and build wheels --- .github/workflows/ci.yml | 2 ++ CHANGELOG.rst | 2 ++ src/isal/_isalmodule.c | 5 +++++ src/isal/igzip_libmodule.c | 5 +++++ src/isal/isal_zlibmodule.c | 4 ++++ 5 files changed, 18 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b886a730..104b11db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,7 @@ jobs: - "3.11" - "3.12" - "3.13" + - "3.13t" - "pypy-3.9" - "pypy-3.10" os: ["ubuntu-latest"] @@ -215,6 +216,7 @@ jobs: run: cibuildwheel --output-dir dist env: CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_*arm64 cp39-macosx_*arm64" # Skip 32 bit and problematic mac builds. + CIBW_ENABLE: "cpython-freethreading" CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }} # Fully test the build wheels again. diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c49acba7..c20a0a5f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,8 @@ version 1.8.0-dev + Include test packages in the source distribution, so source distribution installations can be verified. + Fix an issue where some tests failed because they ignored PYTHONPATH. ++ Enable support for free-threading and build free-threaded wheels for + CPython 3.13. version 1.7.2 ----------------- diff --git a/src/isal/_isalmodule.c b/src/isal/_isalmodule.c index a429d3a7..55f43917 100644 --- a/src/isal/_isalmodule.c +++ b/src/isal/_isalmodule.c @@ -31,6 +31,11 @@ PyInit__isal(void) if (m == NULL) { return NULL; } + +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + PyModule_AddIntMacro(m, ISAL_MAJOR_VERSION); PyModule_AddIntMacro(m, ISAL_MINOR_VERSION); PyModule_AddIntMacro(m, ISAL_PATCH_VERSION); diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index de94ab01..d498145f 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -617,6 +617,11 @@ PyInit_igzip_lib(void) if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + + IsalError = PyErr_NewException("igzip_lib.IsalError", NULL, NULL); if (IsalError == NULL) { return NULL; diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 67547be9..3a280ec8 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -2183,6 +2183,10 @@ PyInit_isal_zlib(void) return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + PyObject *igzip_lib_module = PyImport_ImportModule("isal.igzip_lib"); if (igzip_lib_module == NULL) { return NULL; From 5ef868bd29559dd1647230622bae3c8224f27738 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 11:40:33 +0200 Subject: [PATCH 105/118] Fix flush implementation --- src/isal/igzip_threaded.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index 7f1c94fc..355b7f72 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -316,7 +316,7 @@ def write(self, b) -> int: self.input_queues[worker_index].put((data, zdict)) return len(data) - def _end_gzip_stream(self): + def flush(self): self._check_closed() # Wait for all data to be compressed for in_q in self.input_queues: @@ -324,22 +324,18 @@ def _end_gzip_stream(self): # Wait for all data to be written for out_q in self.output_queues: out_q.join() - # Write an empty deflate block with a lost block marker. + self.raw.flush() + + def close(self) -> None: + if self._closed: + return + self.flush() self.raw.write(isal_zlib.compress(b"", wbits=-15)) trailer = struct.pack(" None: - if self._closed: - return - self._end_gzip_stream() self.stop() if self.exception: self.raw.close() From 43983269f56a911e84fce09b56cfb95765f6334f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 14:22:54 +0200 Subject: [PATCH 106/118] Add a test for correctly flushing data to disk --- tests/test_igzip_threaded.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 41c61bfe..6331182b 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -12,6 +12,7 @@ import subprocess import sys import tempfile +import zlib from pathlib import Path from isal import igzip_threaded @@ -243,15 +244,28 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): @pytest.mark.parametrize("threads", [1, 2]) def test_flush(tmp_path, threads): + empty_block_end = b"\x00\x00\xff\xff" + deflate_last_block = zlib.compress(b"", wbits=-15) test_file = tmp_path / "output.gz" with igzip_threaded.open(test_file, "wb", threads=threads) as f: f.write(b"1") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"1" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + # Cut off gzip header and end data with an explicit last block to + # test if the data was compressed correctly. + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"1" f.write(b"2") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"12" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"12" f.write(b"3") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"123" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"123" assert gzip.decompress(test_file.read_bytes()) == b"123" From 85e9d4853ca9d67d7b206c71428fc597b31549c6 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 14:26:24 +0200 Subject: [PATCH 107/118] Add multithreaded flushing fix to the CHANGELOG --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c49acba7..508abc46 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,10 @@ Changelog version 1.8.0-dev ----------------- + Python 3.8 is no longer supported. ++ Fix an issue where flushing using igzip_threaded caused a gzip end of stream + and started a new gzip stream. In essence creating a concatenated gzip + stream. Now it is in concordance with how single threaded gzip streams + are flushed using Z_SYNC_FLUSH. + Change build backend to setuptools-scm which is more commonly used and supported. + Include test packages in the source distribution, so source distribution From 0fef0a9bd44689a51d0e32fd51f07708e96e3012 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 14:31:31 +0200 Subject: [PATCH 108/118] Fix test for older versions of python --- tests/test_igzip_threaded.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 6331182b..d2aee567 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -245,7 +245,8 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): @pytest.mark.parametrize("threads", [1, 2]) def test_flush(tmp_path, threads): empty_block_end = b"\x00\x00\xff\xff" - deflate_last_block = zlib.compress(b"", wbits=-15) + compressobj = zlib.compressobj(wbits=-15) + deflate_last_block = compressobj.compress(b"") + compressobj.flush() test_file = tmp_path / "output.gz" with igzip_threaded.open(test_file, "wb", threads=threads) as f: f.write(b"1") From eb3e5c23bbea2e1d91e0d2cb50114c2522870d7f Mon Sep 17 00:00:00 2001 From: Mattias Ellert Date: Wed, 2 Jul 2025 09:49:25 +0200 Subject: [PATCH 109/118] Fix a typo in a comment --- tests/test_gzip_compliance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index bd11451d..5d71577c 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -850,7 +850,7 @@ def test_decompress_infile_outfile(self): # If the env_vars argument to assert_python_failure or assert_python_ok # is empty the test will run in isolated mode (-I) which means that the # PYTHONPATH environment variable will be ignored and the test fails - # because the isal module can not be found, or the test is run usung the + # because the isal module can not be found, or the test is run using the # system installed version of the module instead of the newly built # module that should be tested. # From ffd757b260106ae2d3a27894b0bfed98622f1267 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 8 Sep 2025 15:04:22 +0300 Subject: [PATCH 110/118] Add test for free threading that compresses and decompresses bytes --- tests/test_freethreading.py | 110 ++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 tests/test_freethreading.py diff --git a/tests/test_freethreading.py b/tests/test_freethreading.py new file mode 100644 index 00000000..a338bcca --- /dev/null +++ b/tests/test_freethreading.py @@ -0,0 +1,110 @@ +import concurrent.futures +import random +import string +import threading + +import pytest + +from isal import isal_zlib, igzip_lib + + +HAMLET_SCENE = b""" +LAERTES + + O, fear me not. + I stay too long: but here my father comes. + + Enter POLONIUS + + A double blessing is a double grace, + Occasion smiles upon a second leave. + +LORD POLONIUS + + Yet here, Laertes! aboard, aboard, for shame! + The wind sits in the shoulder of your sail, + And you are stay'd for. There; my blessing with thee! + And these few precepts in thy memory + See thou character. Give thy thoughts no tongue, + Nor any unproportioned thought his act. + Be thou familiar, but by no means vulgar. + Those friends thou hast, and their adoption tried, + Grapple them to thy soul with hoops of steel; + But do not dull thy palm with entertainment + Of each new-hatch'd, unfledged comrade. Beware + Of entrance to a quarrel, but being in, + Bear't that the opposed may beware of thee. + Give every man thy ear, but few thy voice; + Take each man's censure, but reserve thy judgment. + Costly thy habit as thy purse can buy, + But not express'd in fancy; rich, not gaudy; + For the apparel oft proclaims the man, + And they in France of the best rank and station + Are of a most select and generous chief in that. + Neither a borrower nor a lender be; + For loan oft loses both itself and friend, + And borrowing dulls the edge of husbandry. + This above all: to thine ownself be true, + And it must follow, as the night the day, + Thou canst not then be false to any man. + Farewell: my blessing season this in thee! + +LAERTES + + Most humbly do I take my leave, my lord. + +LORD POLONIUS + + The time invites you; go; your servants tend. + +LAERTES + + Farewell, Ophelia; and remember well + What I have said to you. + +OPHELIA + + 'Tis in my memory lock'd, + And you yourself shall keep the key of it. + +LAERTES + + Farewell. +""" + +NUM_THREADS = 10 +NUM_ITERATIONS = 20 +NUM_JOBS = 50 # To simulate 50 jobs running in 10 threads +barrier = threading.Barrier(parties=NUM_THREADS) + + +def isal_compress_decompress(compress, decompress): + for _ in range(NUM_ITERATIONS): + barrier.wait() + x = compress(HAMLET_SCENE) + assert decompress(x) == HAMLET_SCENE + + length = len(HAMLET_SCENE) + hamlet_random = HAMLET_SCENE + b"".join( + [s.encode() for s in random.choices(string.printable, k=length)] + ) + barrier.wait() + x = compress(hamlet_random) + assert decompress(x) == hamlet_random + + +@pytest.mark.parametrize( + "compress,decompress", + [ + pytest.param(isal_zlib.compress, isal_zlib.decompress, id="zlib"), + pytest.param(igzip_lib.compress, igzip_lib.decompress, id="igzip"), + ] +) +def test_isal_compress_decompress_threaded(compress, decompress): + with concurrent.futures.ThreadPoolExecutor(NUM_THREADS) as executor: + futures = [ + executor.submit(isal_compress_decompress, compress, decompress) + for _ in range(NUM_JOBS) + ] + for future in concurrent.futures.as_completed(futures): + future.result() # To fire assertion error if there is one From a3a3a1597795a8b95a7900694976e9c1b9d4aab8 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 8 Sep 2025 15:07:30 +0300 Subject: [PATCH 111/118] Test 3.14 on CI --- .github/workflows/ci.yml | 3 ++- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 104b11db..76e58fbf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,8 @@ jobs: - "3.12" - "3.13" - "3.13t" + - "3.14-dev" + - "3.14t-dev" - "pypy-3.9" - "pypy-3.10" os: ["ubuntu-latest"] @@ -216,7 +218,6 @@ jobs: run: cibuildwheel --output-dir dist env: CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_*arm64 cp39-macosx_*arm64" # Skip 32 bit and problematic mac builds. - CIBW_ENABLE: "cpython-freethreading" CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }} # Fully test the build wheels again. diff --git a/pyproject.toml b/pyproject.toml index 8405984e..0a95142a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", From b23507830a8061e855e32bf179befcbd1a6ea653 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 14:32:38 +0200 Subject: [PATCH 112/118] Remove python 3.9 support and add 3.14 support --- .github/workflows/ci.yml | 21 +++++++++++---------- CHANGELOG.rst | 3 ++- pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b886a730..9e95e301 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install tox run: pip install tox - name: Lint @@ -36,10 +36,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install isal run: sudo apt-get install libisal-dev - name: Install tox and upgrade setuptools and pip @@ -54,21 +54,21 @@ jobs: strategy: matrix: python-version: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - - "pypy-3.9" + - "3.14" - "pypy-3.10" + - "pypy-3.11" os: ["ubuntu-latest"] include: - os: "macos-13" - python-version: "3.9" - - os: "macos-14" + python-version: "3.10" + - os: "macos-latest" python-version: "3.10" - os: "windows-latest" - python-version: "3.9" + python-version: "3.10" steps: - uses: actions/checkout@v4 with: @@ -77,6 +77,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install tox and upgrade setuptools run: pip install --upgrade tox setuptools - name: Install build dependencies (Linux) # Yasm in pypa/manylinux images. @@ -104,7 +105,7 @@ jobs: strategy: matrix: python_version: - - "3.9" + - "3.10" steps: - uses: actions/checkout@v4 with: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 508abc46..9f430fa1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,7 +10,8 @@ Changelog version 1.8.0-dev ----------------- -+ Python 3.8 is no longer supported. ++ Python 3.14 is supported. ++ Python 3.8 and 3.9 are no longer supported. + Fix an issue where flushing using igzip_threaded caused a gzip end of stream and started a new gzip stream. In essence creating a concatenated gzip stream. Now it is in concordance with how single threaded gzip streams diff --git a/pyproject.toml b/pyproject.toml index 8405984e..9fb66063 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,11 +17,11 @@ requires-python = ">=3.9" # Because of setuptools version classifiers = [ "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", From 474783e98241eed2433b5194a5219ebf7a5670f7 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 14:52:35 +0200 Subject: [PATCH 113/118] Only support 3.14 for threaded builds. --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5d9d1581..e0d37375 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,7 +22,7 @@ version 1.8.0-dev installations can be verified. + Fix an issue where some tests failed because they ignored PYTHONPATH. + Enable support for free-threading and build free-threaded wheels for - CPython 3.13. + CPython 3.14. version 1.7.2 ----------------- From 0b9574a16d443ce99de8307215e3c02aa585a16c Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 14:58:56 +0200 Subject: [PATCH 114/118] Fix linting issues --- tests/test_freethreading.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_freethreading.py b/tests/test_freethreading.py index a338bcca..8e0ae092 100644 --- a/tests/test_freethreading.py +++ b/tests/test_freethreading.py @@ -3,9 +3,9 @@ import string import threading -import pytest +from isal import igzip_lib, isal_zlib -from isal import isal_zlib, igzip_lib +import pytest HAMLET_SCENE = b""" From 9492937925f55b7def66d434f73190bf2dcb024b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 15:00:48 +0200 Subject: [PATCH 115/118] Update test_freethreading.py Remove trailing whitespace --- tests/test_freethreading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_freethreading.py b/tests/test_freethreading.py index 8e0ae092..10d1d53b 100644 --- a/tests/test_freethreading.py +++ b/tests/test_freethreading.py @@ -3,7 +3,7 @@ import string import threading -from isal import igzip_lib, isal_zlib +from isal import igzip_lib, isal_zlib import pytest From 8e9681d3674b0bd3ffcaf8cff8081edb5231f02a Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 15:03:40 +0200 Subject: [PATCH 116/118] Credit where it's due --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e0d37375..0cd0f65c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,7 +22,7 @@ version 1.8.0-dev installations can be verified. + Fix an issue where some tests failed because they ignored PYTHONPATH. + Enable support for free-threading and build free-threaded wheels for - CPython 3.14. + CPython 3.14. Thanks to @lysnikolaou and @ngoldbaum. version 1.7.2 ----------------- From ddcc9667c2d618df31352c3021959895a2fc1db0 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 10 Sep 2025 09:13:43 +0200 Subject: [PATCH 117/118] Prepare release 1.8.0 --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0cd0f65c..36e20a2d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,7 +8,7 @@ Changelog .. that users understand how the changes affect the new version. -version 1.8.0-dev +version 1.8.0 ----------------- + Python 3.14 is supported. + Python 3.8 and 3.9 are no longer supported. From bcaaa9b0731c76267ef662ac559956004c17db7b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 10 Sep 2025 10:33:21 +0200 Subject: [PATCH 118/118] Use a valid documentation URL --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9fb66063..c1cbce68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ "Operating System :: Microsoft :: Windows", ] urls.homepage = "https://github.com/pycompression/python-isal" -urls.documentation = "python-isal.readthedocs.io" +urls.documentation = "https://python-isal.readthedocs.io" [tool.setuptools_scm] version_file = "src/isal/_version.py"