diff --git a/.github/release_checklist.md b/.github/release_checklist.md index c47123ba..8ae899dc 100644 --- a/.github/release_checklist.md +++ b/.github/release_checklist.md @@ -2,15 +2,13 @@ Release checklist - [ ] Check outstanding issues on JIRA and Github. - [ ] Check [latest documentation](https://python-isal.readthedocs.io/en/latest/) looks fine. - [ ] Create a release branch. - - [ ] Set version to a stable number. - [ ] Change current development version in `CHANGELOG.rst` to stable version. - - [ ] Change the version in `__init__.py` +- [ ] Check if the address sanitizer does not find any problems using `tox -e asan` - [ ] Merge the release branch into `main`. - [ ] Created an annotated tag with the stable version number. Include changes from CHANGELOG.rst. - [ ] Push tag to remote. This triggers the wheel/sdist build on github CI. - [ ] merge `main` branch back into `develop`. -- [ ] Add updated version number to develop. (`setup.py` and `src/isal/__init__.py`) - [ ] Build the new tag on readthedocs. Only build the last patch version of each minor version. So `1.1.1` and `1.2.0` but not `1.1.0`, `1.1.1` and `1.2.0`. - [ ] Create a new release on github. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 338a177c..955ca769 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,6 @@ name: Continous integration on: pull_request: - paths-ignore: - - 'docs/**' - - '*.rst' push: branches: - develop @@ -16,13 +13,13 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 - uses: actions/setup-python@v2.2.1 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.10" - name: Install tox run: pip install tox - name: Lint @@ -36,13 +33,13 @@ jobs: - twine_check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 - uses: actions/setup-python@v2.2.1 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.10" - name: Install isal run: sudo apt-get install libisal-dev - name: Install tox and upgrade setuptools and pip @@ -57,27 +54,31 @@ jobs: strategy: matrix: python-version: - - "3.8" - - "3.9" - "3.10" - "3.11" - "3.12" - - "pypy-3.9" + - "3.13" + - "3.14" + - "3.14t" - "pypy-3.10" + - "pypy-3.11" os: ["ubuntu-latest"] include: + - os: "macos-13" + python-version: "3.10" - os: "macos-latest" - python-version: 3.8 + python-version: "3.10" - os: "windows-latest" - python-version: 3.8 + python-version: "3.10" steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2.2.1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install tox and upgrade setuptools run: pip install --upgrade tox setuptools - name: Install build dependencies (Linux) # Yasm in pypa/manylinux images. @@ -89,10 +90,10 @@ jobs: run: brew install nasm if: runner.os == 'macOS' - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Install nasm (Windows) - uses: ilammy/setup-nasm@v1.2.0 + uses: ilammy/setup-nasm@v1 if: runner.os == 'Windows' - name: Run tests run: tox @@ -105,18 +106,22 @@ jobs: strategy: matrix: python_version: - - "3.8" + - "3.10" steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - - uses: uraimo/run-on-arch-action@v2.5.0 + - uses: uraimo/run-on-arch-action@v3 name: Build & run test with: arch: none distro: none base_image: "--platform=linux/arm64 quay.io/pypa/manylinux2014_aarch64" + # versioningit needs an accessible git repository but the container + # is run as root, which is different from the repository user. + # use git config to override this. run: |- + git config --global --add safe.directory $PWD CFLAGS="-DNDEBUG -g0" python${{matrix.python_version}} -m pip install . pytest python${{matrix.python_version}} -m pytest tests @@ -131,23 +136,23 @@ jobs: shell: bash -l {0} strategy: matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest", "macos-13", "windows-latest"] python_version: [ "python" ] include: - os: "ubuntu-latest" python_version: "pypy" steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - name: Install miniconda. - uses: conda-incubator/setup-miniconda@v2.0.1 # https://github.com/conda-incubator/setup-miniconda. + uses: conda-incubator/setup-miniconda@v3 # https://github.com/conda-incubator/setup-miniconda. with: channels: conda-forge,defaults - name: Install requirements (universal) run: conda install isa-l ${{ matrix.python_version}} tox - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Run tests (dynamic link) run: tox @@ -167,7 +172,8 @@ jobs: matrix: os: - ubuntu-latest - - macos-latest + - macos-13 + - macos-14 - windows-latest cibw_archs_linux: ["x86_64"] cibw_before_all_linux: @@ -185,10 +191,11 @@ jobs: cibw_archs_linux: "aarch64" cibw_before_all_linux: "true" # The true command exits with 0 steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - - uses: actions/setup-python@v2 + fetch-depth: 0 # Fetch everything to get accurately versioned tag. + - uses: actions/setup-python@v2 # Some issues where caused by higher versions. name: Install Python - name: Install cibuildwheel twine wheel run: python -m pip install cibuildwheel twine wheel @@ -196,20 +203,20 @@ jobs: run: brew install nasm if: runner.os == 'macOS' - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Install nasm (Windows) - uses: ilammy/setup-nasm@v1.2.0 + uses: ilammy/setup-nasm@v1 if: runner.os == 'Windows' - name: Set up QEMU if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'aarch64'}} - uses: docker/setup-qemu-action@v1.0.1 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Build wheels run: cibuildwheel --output-dir dist env: - CIBW_SKIP: "*-win32 *-manylinux_i686" # Skip 32 bit + CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_*arm64 cp39-macosx_*arm64" # Skip 32 bit and problematic mac builds. CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }} # Fully test the build wheels again. @@ -237,34 +244,34 @@ jobs: CIBW_ENVIRONMENT_LINUX: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=/tmp/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" CIBW_ENVIRONMENT_WINDOWS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}\build_cache CIBW_ENVIRONMENT_MACOS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" - name: Build sdist if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'x86_64'}} run: | pip install build python -m build --sdist - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: - name: "dist" + name: "dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cibw_archs_linux }}" path: "dist/" - name: Publish package to TestPyPI # pypa/gh-action-pypi-publish@master does not work on OSX # Alpha, Beta and dev releases contain a - in the tag. if: contains(github.ref, '-') && startsWith(github.ref, 'refs/tags') - run: twine upload --repository-url https://test.pypi.org/legacy/ dist/* + run: twine upload --skip-existing --repository-url https://test.pypi.org/legacy/ dist/* env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} - name: Publish package to PyPI if: "!contains(github.ref, '-') && startsWith(github.ref, 'refs/tags')" - run: twine upload dist/* + run: twine upload --skip-existing dist/* env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index b6e47617..8e29d4aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +src/isal/_version.py + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/.gitmodules b/.gitmodules index 11a84d8b..d79a05a7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "src/isal/isa-l"] path = src/isal/isa-l - url = https://github.com/rhpvorderman/isa-l.git + url = https://github.com/intel/isa-l.git diff --git a/.readthedocs.yml b/.readthedocs.yml index 2866edaf..910b7650 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,7 +3,16 @@ formats: [] # Do not build epub and pdf python: install: - - method: pip - path: . -conda: - environment: docs/conda-environment.yml \ No newline at end of file + - requirements: "requirements-docs.txt" + - method: "pip" + path: "." + +sphinx: + configuration: docs/conf.py + +build: + os: "ubuntu-22.04" + tools: + python: "3" + apt_packages: + - libisal-dev diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0f1eaff3..36e20a2d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,72 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. + +version 1.8.0 +----------------- ++ Python 3.14 is supported. ++ Python 3.8 and 3.9 are no longer supported. ++ Fix an issue where flushing using igzip_threaded caused a gzip end of stream + and started a new gzip stream. In essence creating a concatenated gzip + stream. Now it is in concordance with how single threaded gzip streams + are flushed using Z_SYNC_FLUSH. ++ Change build backend to setuptools-scm which is more commonly used and + supported. ++ Include test packages in the source distribution, so source distribution + installations can be verified. ++ Fix an issue where some tests failed because they ignored PYTHONPATH. ++ Enable support for free-threading and build free-threaded wheels for + CPython 3.14. Thanks to @lysnikolaou and @ngoldbaum. + +version 1.7.2 +----------------- ++ Use upstream ISA-L version 2.31.1 which includes patches to make + installation on MacOS ARM64 possible. ++ Fix a bug where bytes were copied in the wrong order on big endian + architectures. Fixes test failures on s390x. ++ Enable building on GNU/Hurd platforms. + +version 1.7.1 +----------------- ++ Fix a bug where flushing files when writing in threaded mode did not work + properly. ++ Prevent threaded opening from blocking python exit when an error is thrown + in the calling thread. + +version 1.7.0 +----------------- ++ Include a patched ISA-L version 2.31. The applied patches make compilation + and wheelbuilding on MacOS ARM64 possible. ++ Fix a bug where READ and WRITE in isal.igzip were inconsistent with the + values in gzip on Python 3.13 ++ Small simplifications to the ``igzip.compress`` function, which should lead + to less overhead. + +version 1.6.1 +----------------- ++ Fix a bug where streams that were passed to igzip_threaded.open where closed. + +version 1.6.0 +----------------- ++ Fix a bug where compression levels for IGzipFile where checked in read mode. ++ Update statically linked ISA-L release to 2.31.0 ++ Fix an error that occurred in the ``__close__`` function when a threaded + writer was initialized with incorrect parameters. + +version 1.5.3 +----------------- ++ Fix a bug where append mode would not work when using + ``igzip_threaded.open``. + +version 1.5.2 +----------------- ++ Fix a bug where a filehandle remained opened when ``igzip_threaded.open`` + was used for writing with a wrong compression level. ++ Fix a memory leak that occurred when an error was thrown for a gzip header + with the wrong magic numbers. ++ Fix a memory leak that occurred when isal_zlib.decompressobj was given a + wrong wbits value. + version 1.5.1 ----------------- + Fix a memory leak in the GzipReader.readall implementation. diff --git a/MANIFEST.in b/MANIFEST.in index 3ef9dc16..4a48e873 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,9 @@ graft src/isal/isa-l include src/isal/*.h +prune docs +prune .github +exclude .git* +prune benchmark_scripts +exclude requirements-docs.txt +exclude codecov.yml +exclude .readthedocs.yml diff --git a/README.rst b/README.rst index fc3d5195..734b2f74 100644 --- a/README.rst +++ b/README.rst @@ -101,8 +101,6 @@ Installation is supported on Linux, Windows and MacOS. For more advanced installation options check the `documentation `_. -.. _differences-with-zlib-and-gzip-modules: - python-isal as a dependency in your project ------------------------------------------- @@ -125,6 +123,8 @@ your project please list a python-isal dependency as follows. .. dependency end +.. _differences-with-zlib-and-gzip-modules: + Differences with zlib and gzip modules -------------------------------------- @@ -166,6 +166,28 @@ are also very welcome. Please report them on the `github issue tracker .. contributing end +Development +----------- +.. development start + +The repository needs to be cloned recursively to make sure the +`ISA-L `_ repository is checked out: +``git clone --recursive https://github.com/pycompression/python-isal.git``. If +the repository is already checked out you can use ``git submodule update --init``. + +Patches should be made on a feature branch. To run the testing install ``tox`` +with ``pip install tox`` and run the commands ``tox -e lint`` and +``tox``. That will run most of the testing that is also performed by the CI. +For changes to the documentation run ``tox -e docs``. For changes to the C +code please also run ``tox -e asan`` to check for memory leaks. This requires +libasan to be installed. + +Building requires the +`ISA-L build requirements `_ +as well. + +.. development end + Acknowledgements ---------------- diff --git a/docs/conda-environment.yml b/docs/conda-environment.yml deleted file mode 100644 index a23cb482..00000000 --- a/docs/conda-environment.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: rtd -channels: - - conda-forge - - defaults -dependencies: - - isa-l - - python >=3.6 - - sphinx - - setuptools - - pip: - - sphinx-rtd-theme - - sphinx-argparse \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 1f6b7297..f2ec9dd2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,27 +4,20 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -from distutils.dist import DistributionMetadata -from pathlib import Path - -import pkg_resources +from isal import __version__ # -- Project information ----------------------------------------------------- # Get package information from the installed package. -package = pkg_resources.get_distribution("isal") -metadata_file = Path(package.egg_info) / Path(package.PKG_INFO) -metadata = DistributionMetadata(path=str(metadata_file)) project = 'python-isal' copyright = '2020, Leiden University Medical Center' author = 'Leiden University Medical Center' # The short X.Y version -version = package.parsed_version.base_version +version = __version__ # The full version, including alpha/beta/rc tags -release = package.version - +release = __version__ # -- General configuration --------------------------------------------------- @@ -49,9 +42,6 @@ # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' -html_theme_options = dict( - display_version=True, -) # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/index.rst b/docs/index.rst index 2d540938..f06feb9a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -158,6 +158,13 @@ Contributing :start-after: .. contributing start :end-before: .. contributing end +=========== +Development +=========== +.. include:: includes/README.rst + :start-after: .. development start + :end-before: .. development end + ================ Acknowledgements ================ diff --git a/pyproject.toml b/pyproject.toml index 62f5f82c..c1cbce68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,61 @@ [build-system] -requires = ["setuptools>=51", "wheel"] +requires = ["setuptools>=77", "setuptools-scm>=8"] build-backend = "setuptools.build_meta" + +[project] +name = "isal" +dynamic = ["version"] +description = """ +Faster zlib and gzip compatible compression and decompression by providing \ +python bindings for the ISA-L ibrary.""" +license="PSF-2.0" +keywords=["isal", "isa-l", "compression", "deflate", "gzip", "igzip"] +authors = [{name = "Leiden University Medical Center"}, + {email = "r.h.p.vorderman@lumc.nl"}] +readme = "README.rst" +requires-python = ">=3.9" # Because of setuptools version +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: C", + "Development Status :: 5 - Production/Stable", + "Topic :: System :: Archiving :: Compression", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", +] +urls.homepage = "https://github.com/pycompression/python-isal" +urls.documentation = "https://python-isal.readthedocs.io" + +[tool.setuptools_scm] +version_file = "src/isal/_version.py" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["isal"] + +[tool.setuptools.package-data] +isal = ['*.pyi', 'py.typed', 'isa-l/LICENSE', 'isa-l/README.md', 'isa-l/Release_notes.txt'] +[tool.setuptools.exclude-package-data] +isal = [ + "*.c", + "*.h", + "isa-l/*/*", + "isa-l/Mak*", + "isa-l/.*", + "isa-l/autogen.sh", + "isa-l/Doxyfile", + "isa-l/CONTRIBUTING.md", + "isa-l/SECURITY.md", + "isa-l/configure.ac", + "isa-l/isa-l.*", + "isa-l/libisal.pc.in", + "isa-l/make.inc", +] diff --git a/requirements-docs.txt b/requirements-docs.txt index 051c278d..8fb9ea6c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,4 +1,4 @@ -sphinx -setuptools +# https://github.com/sphinx-doc/sphinx/issues/13415 +sphinx <8 sphinx-rtd-theme -sphinx-argparse \ No newline at end of file +sphinx-argparse diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 256134e1..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -license_files=LICENSE \ No newline at end of file diff --git a/setup.py b/setup.py index 9bc284fd..1ed64c2d 100644 --- a/setup.py +++ b/setup.py @@ -15,13 +15,17 @@ import tempfile from pathlib import Path -from setuptools import Extension, find_packages, setup +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext ISA_L_SOURCE = os.path.join("src", "isal", "isa-l") +SYSTEM_IS_BSD = (sys.platform.startswith("freebsd") or + sys.platform.startswith("netbsd")) SYSTEM_IS_UNIX = (sys.platform.startswith("linux") or - sys.platform.startswith("darwin")) + sys.platform.startswith("darwin") or + sys.platform.startswith("gnu") or + SYSTEM_IS_BSD) SYSTEM_IS_WINDOWS = sys.platform.startswith("win") # Since pip builds in a temp directory by default, setting a fixed file in @@ -33,16 +37,11 @@ DEFAULT_CACHE_FILE)) EXTENSIONS = [ - Extension("isal.isal_zlib", ["src/isal/isal_zlibmodule.c"]), - Extension("isal.igzip_lib", ["src/isal/igzip_libmodule.c"]), + Extension("isal.isal_zlib", ["src/isal/isal_zlibmodule.c"]), + Extension("isal.igzip_lib", ["src/isal/igzip_libmodule.c"]), + Extension("isal._isal", ["src/isal/_isalmodule.c"]), ] -# This does not add the extension on windows for dynamic linking. The required -# header file might be missing. -if not (SYSTEM_IS_WINDOWS and - os.getenv("PYTHON_ISAL_LINK_DYNAMIC") is not None): - EXTENSIONS.append(Extension("isal._isal", ["src/isal/_isalmodule.c"])) - class BuildIsalExt(build_ext): def build_extension(self, ext): @@ -118,7 +117,10 @@ def build_isa_l(): cflags_param = "CFLAGS_aarch64" else: cflags_param = "CFLAGS_" - subprocess.run(["make", "-j", str(cpu_count), "-f", "Makefile.unx", + make_cmd = "make" + if SYSTEM_IS_BSD: + make_cmd = "gmake" + subprocess.run([make_cmd, "-j", str(cpu_count), "-f", "Makefile.unx", "isa-l.h", "bin/isa-l.a", f"{cflags_param}={build_env.get('CFLAGS', '')}"], **run_args) @@ -134,45 +136,6 @@ def build_isa_l(): setup( - name="isal", - version="1.5.1", - description="Faster zlib and gzip compatible compression and " - "decompression by providing python bindings for the ISA-L " - "library.", - author="Leiden University Medical Center", - author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now - long_description=Path("README.rst").read_text(), - long_description_content_type="text/x-rst", cmdclass={"build_ext": BuildIsalExt}, - license="PSF-2.0", - keywords="isal isa-l compression deflate gzip igzip", - zip_safe=False, - packages=find_packages('src'), - package_dir={'': 'src'}, - package_data={'isal': ['*.pyi', 'py.typed', - # Include isa-l LICENSE and other relevant files - # with the binary distribution. - 'isa-l/LICENSE', 'isa-l/README.md', - 'isa-l/Release_notes.txt']}, - url="https://github.com/pycompression/python-isal", - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: C", - "Development Status :: 5 - Production/Stable", - "Topic :: System :: Archiving :: Compression", - "License :: OSI Approved :: Python Software Foundation License", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - ], - python_requires=">=3.8", # BadGzipFile imported ext_modules=EXTENSIONS ) diff --git a/src/isal/__init__.py b/src/isal/__init__.py index 9306adfc..7addbeef 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -5,19 +5,9 @@ # This file is part of python-isal which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. -from typing import Optional - -try: - from . import _isal - ISAL_MAJOR_VERSION: Optional[int] = _isal.ISAL_MAJOR_VERSION - ISAL_MINOR_VERSION: Optional[int] = _isal.ISAL_MINOR_VERSION - ISAL_PATCH_VERSION: Optional[int] = _isal.ISAL_PATCH_VERSION - ISAL_VERSION: Optional[str] = _isal.ISAL_VERSION -except ImportError: - ISAL_MAJOR_VERSION = None - ISAL_MINOR_VERSION = None - ISAL_PATCH_VERSION = None - ISAL_VERSION = None +from ._isal import (ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION, + ISAL_VERSION) +from ._version import __version__ __all__ = [ "ISAL_MAJOR_VERSION", @@ -26,5 +16,3 @@ "ISAL_VERSION", "__version__" ] - -__version__ = "1.5.1" diff --git a/src/isal/_isalmodule.c b/src/isal/_isalmodule.c index a429d3a7..55f43917 100644 --- a/src/isal/_isalmodule.c +++ b/src/isal/_isalmodule.c @@ -31,6 +31,11 @@ PyInit__isal(void) if (m == NULL) { return NULL; } + +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + PyModule_AddIntMacro(m, ISAL_MAJOR_VERSION); PyModule_AddIntMacro(m, ISAL_MINOR_VERSION); PyModule_AddIntMacro(m, ISAL_PATCH_VERSION); diff --git a/src/isal/_version.pyi b/src/isal/_version.pyi new file mode 100644 index 00000000..e59e7032 --- /dev/null +++ b/src/isal/_version.pyi @@ -0,0 +1,8 @@ +# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +# 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 +# Python Software Foundation; All Rights Reserved + +# This file is part of python-isal which is distributed under the +# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. + +__version__: str diff --git a/src/isal/igzip.py b/src/isal/igzip.py index 5b942862..af1b4712 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -9,11 +9,7 @@ # Changes compared to CPython: # - Subclassed GzipFile to IGzipFile. Methods that included calls to zlib have # been overwritten with the same methods, but now calling to isal_zlib. -# - _GzipReader uses a igzip_lib.IgzipDecompressor. This Decompressor is -# derived from the BZ2Decompressor as such it does not produce an unconsumed -# tail but keeps the read data internally. This prevents unnecessary copying -# of data. To accomodate this, the read method has been rewritten. -# - _GzipReader._add_read_data uses isal_zlib.crc32 instead of zlib.crc32. +# - _GzipReader is implemented in C in isal_zlib and allows dropping the GIL. # - Gzip.compress does not use a GzipFile to compress in memory, but creates a # simple header using _create_simple_gzip_header and compresses the data with # igzip_lib.compress using the DECOMP_GZIP_NO_HDR flag. This change was @@ -33,6 +29,7 @@ import gzip import io import os +import shutil import struct import sys import time @@ -55,7 +52,8 @@ READ_BUFFER_SIZE = 512 * 1024 FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 -READ, WRITE = 1, 2 +READ = gzip.READ +WRITE = gzip.WRITE BadGzipFile = gzip.BadGzipFile # type: ignore @@ -151,11 +149,12 @@ def __init__(self, filename=None, mode=None, If omitted or None, the current time is used. """ if not (isal_zlib.ISAL_BEST_SPEED <= compresslevel - <= isal_zlib.ISAL_BEST_COMPRESSION): + <= isal_zlib.ISAL_BEST_COMPRESSION) and "r" not in mode: raise ValueError( - "Compression level should be between {0} and {1}.".format( - isal_zlib.ISAL_BEST_SPEED, isal_zlib.ISAL_BEST_COMPRESSION - )) + f"Compression level should be between " + f"{isal_zlib.ISAL_BEST_SPEED} and " + f"{isal_zlib.ISAL_BEST_COMPRESSION}, got {compresslevel}." + ) super().__init__(filename, mode, compresslevel, fileobj, mtime) if self.mode == WRITE: self.compress = isal_zlib.compressobj(compresslevel, @@ -223,13 +222,10 @@ def write(self, data): _IGzipReader = _GzipReader -def _create_simple_gzip_header(compresslevel: int, - mtime: Optional[SupportsInt] = None) -> bytes: - """ - Write a simple gzip header with no extra fields. - :param compresslevel: Compresslevel used to determine the xfl bytes. - :param mtime: The mtime (must support conversion to a 32-bit integer). - :return: A bytes object representing the gzip header. +def compress(data, compresslevel: int = _COMPRESS_LEVEL_BEST, *, + mtime: Optional[SupportsInt] = None) -> bytes: + """Compress data in one shot and return the compressed string. + Optional argument is the compression level, in range of 0-3. """ if mtime is None: mtime = time.time() @@ -238,14 +234,7 @@ def _create_simple_gzip_header(compresslevel: int, xfl = 4 if compresslevel == _COMPRESS_LEVEL_FAST else 0 # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra # fields added to header), mtime, xfl and os (255 for unknown OS). - return struct.pack(" None: self.running = False self.worker.join() self.fileobj.close() - self.raw.close() + if self.closefd: + self.raw.close() self._closed = True @property @@ -159,6 +167,12 @@ def closed(self) -> bool: return self._closed +class FlushableBufferedWriter(io.BufferedWriter): + def flush(self): + super().flush() + self.raw.flush() + + class _ThreadedGzipWriter(io.RawIOBase): """ Write a gzip file using multiple threads. @@ -193,15 +207,23 @@ class _ThreadedGzipWriter(io.RawIOBase): compressing and output is handled in one thread. """ def __init__(self, - fp: BinaryIO, + filename, + mode: str = "wb", level: int = isal_zlib.ISAL_DEFAULT_COMPRESSION, threads: int = 1, queue_size: int = 1, block_size: int = 1024 * 1024, ): + # File should be closed during init, so __exit__ method does not + # touch the self.raw value before it is initialized. + self._closed = True + if "t" in mode or "r" in mode: + raise ValueError("Only binary writing is supported") + if "b" not in mode: + mode += "b" self.lock = threading.Lock() + self._calling_thread = threading.current_thread() self.exception: Optional[Exception] = None - self.raw = fp self.level = level self.previous_block = b"" # Deflating random data results in an output a little larger than the @@ -235,6 +257,7 @@ def __init__(self, self._crc = 0 self.running = False self._size = 0 + self.raw, self.closefd = open_as_binary_stream(filename, mode) self._closed = False self._write_gzip_header() self.start() @@ -307,17 +330,19 @@ def close(self) -> None: if self._closed: return self.flush() + self.raw.write(isal_zlib.compress(b"", wbits=-15)) + trailer = struct.pack("read_in >> remainder; + #if PY_BIG_ENDIAN + char *remaining_buffer = (char *)&remaining_bytes; + for (int i = 0; i < n; ++i) { + to[i] = remaining_buffer[7-i]; + } + #else // memcpy works because of little-endianness memcpy(to, &remaining_bytes, n); + #endif return 0; } diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 257260f8..3a280ec8 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -14,14 +14,13 @@ Changes compared to CPython: - Zlib to ISA-L conversion functions were included. - All compression and checksum functions from zlib replaced with ISA-L compatible functions. -- No locks in Compress and Decompress objects. These were deemed unnecessary - as the ISA-L functions do not allocate memory, unlike the zlib - counterparts. - zlib.compress also has a 'wbits' argument. This change was included in Python 3.11. It allows for faster gzip compression by using isal_zlib.compress(data, wbits=31). - Argument parsers were written using th CPython API rather than argument clinic. +- Created a GzipReader class that implements gzip reading in C, reducing a lot + of overhead compared to the gzip.py:_GzipReader class. */ #include "isal_shared.h" @@ -267,7 +266,7 @@ PyDoc_STRVAR(isal_zlib_crc32_combine__doc__, " crc2\n" " the second crc32 checksum\n" " crc2_length\n" -" the lenght of the data block crc2 was calculated from\n" +" the length of the data block crc2 was calculated from\n" ); @@ -793,6 +792,7 @@ isal_zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict) err = wbits_to_flag_and_hist_bits_inflate(wbits, &hist_bits, &flag); if (err < 0) { PyErr_Format(PyExc_ValueError, "Invalid wbits value: %d", wbits); + Py_DECREF(self); return NULL; } else if (err == 0) { @@ -1643,7 +1643,7 @@ static inline uint32_t load_u32_le(void *address) { static inline uint16_t load_u16_le(void *address) { #if PY_BIG_ENDIAN uint8_t *mem = address; - return mem[0] | (mem[1] << 8) | (mem[2] << 16) | (mem[3] << 24); + return mem[0] | (mem[1] << 8); #else return *(uint16_t *)address; #endif @@ -1683,9 +1683,11 @@ GzipReader_read_into_buffer(GzipReader *self, uint8_t *out_buffer, size_t out_bu if (!(magic1 == 0x1f && magic2 == 0x8b)) { Py_BLOCK_THREADS; + PyObject *magic_obj = PyBytes_FromStringAndSize((char *)current_pos, 2); PyErr_Format(BadGzipFile, "Not a gzipped file (%R)", - PyBytes_FromStringAndSize((char *)current_pos, 2)); + magic_obj); + Py_DECREF(magic_obj); return -1; }; uint8_t method = current_pos[2]; @@ -2014,7 +2016,7 @@ GzipReader_readall(GzipReader *self, PyObject *Py_UNUSED(ignore)) Py_DECREF(chunk_list); return NULL; } - PyObject *ret = _PyBytes_Join(empty_bytes, chunk_list); + PyObject *ret = PyObject_CallMethod(empty_bytes, "join", "O", chunk_list); Py_DECREF(empty_bytes); Py_DECREF(chunk_list); return ret; @@ -2181,6 +2183,10 @@ PyInit_isal_zlib(void) return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + PyObject *igzip_lib_module = PyImport_ImportModule("isal.igzip_lib"); if (igzip_lib_module == NULL) { return NULL; diff --git a/tests/test_compat.py b/tests/test_compat.py index baa86bfe..0ea54f70 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -11,7 +11,6 @@ import gzip import itertools -import os import zlib from pathlib import Path @@ -35,8 +34,6 @@ # Wbits for ZLIB compression, GZIP compression, and RAW compressed streams WBITS_RANGE = list(range(9, 16)) + list(range(25, 32)) + list(range(-15, -8)) -DYNAMICALLY_LINKED = os.getenv("PYTHON_ISAL_LINK_DYNAMIC") is not None - @pytest.mark.parametrize(["data_size", "value"], itertools.product(DATA_SIZES, SEEDS)) @@ -93,8 +90,6 @@ def test_decompress_isal_zlib(data_size, level): @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel"], itertools.product([128 * 1024], range(4), WBITS_RANGE, range(1, 10))) -@pytest.mark.xfail(condition=DYNAMICALLY_LINKED, - reason="Dynamically linked version may not have patch.") def test_compress_compressobj(data_size, level, wbits, memLevel): data = DATA[:data_size] compressobj = isal_zlib.compressobj(level=level, diff --git a/tests/test_freethreading.py b/tests/test_freethreading.py new file mode 100644 index 00000000..10d1d53b --- /dev/null +++ b/tests/test_freethreading.py @@ -0,0 +1,110 @@ +import concurrent.futures +import random +import string +import threading + +from isal import igzip_lib, isal_zlib + +import pytest + + +HAMLET_SCENE = b""" +LAERTES + + O, fear me not. + I stay too long: but here my father comes. + + Enter POLONIUS + + A double blessing is a double grace, + Occasion smiles upon a second leave. + +LORD POLONIUS + + Yet here, Laertes! aboard, aboard, for shame! + The wind sits in the shoulder of your sail, + And you are stay'd for. There; my blessing with thee! + And these few precepts in thy memory + See thou character. Give thy thoughts no tongue, + Nor any unproportioned thought his act. + Be thou familiar, but by no means vulgar. + Those friends thou hast, and their adoption tried, + Grapple them to thy soul with hoops of steel; + But do not dull thy palm with entertainment + Of each new-hatch'd, unfledged comrade. Beware + Of entrance to a quarrel, but being in, + Bear't that the opposed may beware of thee. + Give every man thy ear, but few thy voice; + Take each man's censure, but reserve thy judgment. + Costly thy habit as thy purse can buy, + But not express'd in fancy; rich, not gaudy; + For the apparel oft proclaims the man, + And they in France of the best rank and station + Are of a most select and generous chief in that. + Neither a borrower nor a lender be; + For loan oft loses both itself and friend, + And borrowing dulls the edge of husbandry. + This above all: to thine ownself be true, + And it must follow, as the night the day, + Thou canst not then be false to any man. + Farewell: my blessing season this in thee! + +LAERTES + + Most humbly do I take my leave, my lord. + +LORD POLONIUS + + The time invites you; go; your servants tend. + +LAERTES + + Farewell, Ophelia; and remember well + What I have said to you. + +OPHELIA + + 'Tis in my memory lock'd, + And you yourself shall keep the key of it. + +LAERTES + + Farewell. +""" + +NUM_THREADS = 10 +NUM_ITERATIONS = 20 +NUM_JOBS = 50 # To simulate 50 jobs running in 10 threads +barrier = threading.Barrier(parties=NUM_THREADS) + + +def isal_compress_decompress(compress, decompress): + for _ in range(NUM_ITERATIONS): + barrier.wait() + x = compress(HAMLET_SCENE) + assert decompress(x) == HAMLET_SCENE + + length = len(HAMLET_SCENE) + hamlet_random = HAMLET_SCENE + b"".join( + [s.encode() for s in random.choices(string.printable, k=length)] + ) + barrier.wait() + x = compress(hamlet_random) + assert decompress(x) == hamlet_random + + +@pytest.mark.parametrize( + "compress,decompress", + [ + pytest.param(isal_zlib.compress, isal_zlib.decompress, id="zlib"), + pytest.param(igzip_lib.compress, igzip_lib.decompress, id="igzip"), + ] +) +def test_isal_compress_decompress_threaded(compress, decompress): + with concurrent.futures.ThreadPoolExecutor(NUM_THREADS) as executor: + futures = [ + executor.submit(isal_compress_decompress, compress, decompress) + for _ in range(NUM_JOBS) + ] + for future in concurrent.futures.as_completed(futures): + future.result() # To fire assertion error if there is one diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index 8439610b..5d71577c 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -456,6 +456,13 @@ def test_with_open(self): else: self.fail("1/0 didn't raise an exception") + def test_read_and_compresslevel(self): + with igzip.GzipFile(self.filename, "wb") as f: + f.write(b"xxx") + with igzip.GzipFile(self.filename, "rb", compresslevel=17) as f: + data = f.read() + assert data == b"xxx" + def test_zero_padded_file(self): with igzip.GzipFile(self.filename, "wb") as f: f.write(data1 * 50) @@ -785,6 +792,13 @@ def test_newline(self): with igzip.open(self.filename, "rt", newline="\r") as f: self.assertEqual(f.readlines(), [uncompressed]) + def test_reading_and_compresslevel(self): + with igzip.open(self.filename, "wb") as f: + f.write(data1) + with igzip.open(self.filename, "rb", compresslevel=17) as f: + text = f.read() + assert text == data1 + def create_and_remove_directory(directory): def decorator(function): @@ -831,9 +845,23 @@ def test_decompress_infile_outfile(self): self.assertTrue(os.path.exists(igzipname)) + # The following tests use assert_python_failure or assert_python_ok. + # + # If the env_vars argument to assert_python_failure or assert_python_ok + # is empty the test will run in isolated mode (-I) which means that the + # PYTHONPATH environment variable will be ignored and the test fails + # because the isal module can not be found, or the test is run using the + # system installed version of the module instead of the newly built + # module that should be tested. + # + # By adding a dummy entry to the env_vars argument the isolated mode is + # not used and the PYTHONPATH environment variable is not ignored and + # the test works as expected. + def test_decompress_infile_outfile_error(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '-d', - 'thisisatest.out') + 'thisisatest.out', + **{'_dummy': '1'}) self.assertEqual(b"filename doesn't end in .gz: 'thisisatest.out'. " b"Cannot determine output filename.", err.strip()) @@ -858,7 +886,8 @@ def test_compress_infile_outfile_default(self): with open(local_testigzip, 'wb') as fp: fp.write(self.data) - rc, out, err = assert_python_ok('-m', 'isal.igzip', local_testigzip) + rc, out, err = assert_python_ok('-m', 'isal.igzip', local_testigzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(igzipname)) self.assertEqual(out, b'') @@ -877,7 +906,8 @@ def test_compress_infile_outfile(self): rc, out, err = assert_python_ok('-m', 'isal.igzip', compress_level, - local_testigzip) + local_testigzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(igzipname)) self.assertEqual(out, b'') @@ -887,7 +917,7 @@ def test_compress_infile_outfile(self): def test_compress_fast_best_are_exclusive(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '--fast', - '--best') + '--best', **{'_dummy': '1'}) self.assertIn( b"error: argument -3/--best: not allowed with argument -0/--fast", err) @@ -895,7 +925,7 @@ def test_compress_fast_best_are_exclusive(self): def test_decompress_cannot_have_flags_compression(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '--fast', - '-d') + '-d', **{'_dummy': '1'}) self.assertIn( b'error: argument -d/--decompress: not allowed with argument ' b'-0/--fast', diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index a0f581c6..d2aee567 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -9,7 +9,10 @@ import io import itertools import os +import subprocess +import sys import tempfile +import zlib from pathlib import Path from isal import igzip_threaded @@ -99,7 +102,7 @@ def test_threaded_write_oversized_block_no_error(threads): @pytest.mark.parametrize("threads", [1, 3]) def test_threaded_write_error(threads): f = igzip_threaded._ThreadedGzipWriter( - fp=io.BytesIO(), level=3, + io.BytesIO(), level=3, threads=threads, block_size=8 * 1024) # Bypass the write method which should not allow blocks larger than # block_size. @@ -139,10 +142,11 @@ def test_writer_not_readable(): def test_writer_wrong_level(): - with pytest.raises(ValueError) as error: - igzip_threaded._ThreadedGzipWriter(io.BytesIO(), level=42) - error.match("Invalid compression level") - error.match("42") + with tempfile.NamedTemporaryFile("wb") as tmp: + with pytest.raises(ValueError) as error: + igzip_threaded.open(tmp.name, mode="wb", compresslevel=42) + error.match("Invalid compression level") + error.match("42") def test_writer_too_low_threads(): @@ -168,3 +172,101 @@ def test_writer_write_after_close(threads): with pytest.raises(ValueError) as error: f.write(b"abc") error.match("closed") + + +def test_igzip_threaded_append(tmp_path): + test_file = tmp_path / "test.txt.gz" + with igzip_threaded.open(test_file, "wb") as f: + f.write(b"AB") + with igzip_threaded.open(test_file, mode="ab") as f: + f.write(b"CD") + with gzip.open(test_file, "rb") as f: + contents = f.read() + assert contents == b"ABCD" + + +def test_igzip_threaded_append_text_mode(tmp_path): + test_file = tmp_path / "test.txt.gz" + with igzip_threaded.open(test_file, "wt") as f: + f.write("AB") + with igzip_threaded.open(test_file, mode="at") as f: + f.write("CD") + with gzip.open(test_file, "rt") as f: + contents = f.read() + assert contents == "ABCD" + + +def test_igzip_threaded_open_compresslevel_and_reading(tmp_path): + test_file = tmp_path / "test.txt.gz" + test_file.write_bytes(gzip.compress(b"thisisatest")) + with igzip_threaded.open(test_file, compresslevel=5) as f: + text = f.read() + assert text == b"thisisatest" + + +def test_threaded_reader_does_not_close_stream(): + test_stream = io.BytesIO() + test_stream.write(gzip.compress(b"thisisatest")) + test_stream.seek(0) + with igzip_threaded.open(test_stream, "rb") as f: + text = f.read() + assert not test_stream.closed + assert text == b"thisisatest" + + +def test_threaded_writer_does_not_close_stream(): + test_stream = io.BytesIO() + with igzip_threaded.open(test_stream, "wb") as f: + f.write(b"thisisatest") + assert not test_stream.closed + test_stream.seek(0) + assert gzip.decompress(test_stream.read()) == b"thisisatest" + + +@pytest.mark.timeout(5) +@pytest.mark.parametrize( + ["mode", "threads"], itertools.product(["rb", "wb"], [1, 2])) +def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): + program = tmp_path / "no_context_manager.py" + test_file = tmp_path / "output.gz" + # Write 40 mb input data to saturate read buffer. Because of the repetitive + # nature the resulting gzip file is very small (~40 KiB). + test_file.write_bytes(gzip.compress(b"test" * (10 * 1024 * 1024))) + with open(program, "wt") as f: + f.write("from isal import igzip_threaded\n") + f.write( + f"f = igzip_threaded.open('{test_file}', " + f"mode='{mode}', threads={threads})\n" + ) + f.write("raise Exception('Error')\n") + subprocess.run([sys.executable, str(program)]) + + +@pytest.mark.parametrize("threads", [1, 2]) +def test_flush(tmp_path, threads): + empty_block_end = b"\x00\x00\xff\xff" + compressobj = zlib.compressobj(wbits=-15) + deflate_last_block = compressobj.compress(b"") + compressobj.flush() + test_file = tmp_path / "output.gz" + with igzip_threaded.open(test_file, "wb", threads=threads) as f: + f.write(b"1") + f.flush() + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + # Cut off gzip header and end data with an explicit last block to + # test if the data was compressed correctly. + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"1" + f.write(b"2") + f.flush() + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"12" + f.write(b"3") + f.flush() + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"123" + assert gzip.decompress(test_file.read_bytes()) == b"123" diff --git a/tests/test_zlib_compliance.py b/tests/test_zlib_compliance.py index 4e2285b4..e2270050 100644 --- a/tests/test_zlib_compliance.py +++ b/tests/test_zlib_compliance.py @@ -40,10 +40,6 @@ class VersionTestCase(unittest.TestCase): - - @unittest.skipIf(os.getenv("PYTHON_ISAL_LINK_DYNAMIC") is not None and - sys.platform.startswith("win"), - "Header file missing on windows") def test_library_version(self): # Test that the major version of the actual library in use matches the # major version that we were compiled against. We can't guarantee that diff --git a/tox.ini b/tox.ini index c94c6690..256ee95e 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,15 @@ commands = coverage html -i coverage xml -i +[testenv:asan] +setenv= + PYTHONDEVMODE=1 + PYTHONMALLOC=malloc + CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer +allowlist_externals=bash +commands= + bash -c 'export LD_PRELOAD=$(gcc -print-file-name=libasan.so) && printenv LD_PRELOAD && python -c "from isal import isal_zlib" && pytest tests' + [testenv:compliance] deps=pytest commands=