Merge pull request #67 from tcalmant/tests-review

Increase test coverage, fix modified UTF-8/__hash__ bugs, containerize CI

Author: tcalmant

.coveragerc

@@ -3,3 +3,13 @@ source = javaobj/
 
 [report]
 include = javaobj/*
+
+[paths]
+# The test suite is run in a container for each supported Python version (see
+# run_tests_containers.sh), where the tree is unpacked in /work. The data
+# files are then combined outside of the containers: this maps the
+# in-container paths back to the local checkout, so a single report covers
+# every version.
+source =
+    javaobj/
+    /work/javaobj/

.github/workflows/build-24.04.yml

@@ -0,0 +1,134 @@
+# Lints the code, runs the tests on every supported Python version, and
+# reports coverage.
+#
+# The tests run inside the matching official ``python:<version>`` container
+# via ./run_tests_containers.sh. This is what lets CI cover Python 2.7 and
+# 3.4-3.7 too -- actions/setup-python cannot provide those on a current
+# runner. javaobj.v3 requires Python 3.12+ and is skipped below that by
+# run_tests.sh.
+
+name: CI Build
+
+on:
+  push:
+    branches: ["main", "master"]
+    tags: "**"
+  pull_request:
+    branches: ["main", "master"]
+
+jobs:
+  lint:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.14"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install flake8
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+  test:
+    needs: lint
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # Every version declared as supported (pyproject.toml classifiers).
+        # Each runs in its matching python:<version> container, which is the
+        # only way to cover 2.7 and 3.4-3.7 on a modern runner.
+        python-version:
+          [
+            "2.7",
+            "3.4",
+            "3.5",
+            "3.6",
+            "3.7",
+            "3.8",
+            "3.9",
+            "3.10",
+            "3.11",
+            "3.12",
+            "3.13",
+            "3.14",
+          ]
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Java
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: "17"
+      - name: Build the Java test fixtures
+        # tests/java/pom.xml targets -source/-target 7, which javac has
+        # refused since JDK 20: Temurin 17 keeps this working. The generated
+        # .ser fixtures are then streamed into every container below, so none
+        # of them need a JDK themselves.
+        working-directory: tests/java
+        run: mvn -B test
+      - name: Run the suite in a Python ${{ matrix.python-version }} container
+        # run_tests_containers.sh streams a clean copy of the tree into the
+        # container and runs run_tests.sh there. The engine is pinned to
+        # Docker (the runner ships both Docker and Podman, and the script
+        # prefers Podman otherwise). JAVAOBJ_NO_MAVEN skips the redundant
+        # in-container Maven call, since the fixtures were just built above.
+        # COVERAGE_OUTPUT_DIR makes it copy the coverage data of this version
+        # out of the container, so the coverage job below can combine them.
+        env:
+          CONTAINER_ENGINE: docker
+          COVERAGE_OUTPUT_DIR: coverage-data
+          ENGINE_RUN_ARGS: "-e JAVAOBJ_NO_MAVEN=1"
+        run: ./run_tests_containers.sh ${{ matrix.python-version }}
+      - name: Keep the coverage data of Python ${{ matrix.python-version }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: coverage-data-${{ matrix.python-version }}
+          path: coverage-data/*.dat
+          if-no-files-found: error
+          retention-days: 1
+
+  coverage:
+    # Every version of the matrix contributes: the report describes the whole
+    # supported range, not just the one version a runner can install natively.
+    needs: test
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.14"
+      - name: Get the coverage data of every version
+        uses: actions/download-artifact@v8
+        with:
+          pattern: coverage-data-*
+          merge-multiple: true
+          path: coverage-data
+      - name: Combine the coverage data
+        # The data files come from containers, where the tree lives in /work:
+        # .coveragerc maps that path back to this checkout.
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install coverage
+          coverage combine coverage-data/*.dat
+          coverage report
+      - name: Coveralls
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+        run: |
+          pip install coveralls
+          coveralls

.github/workflows/ci-build.yml

@@ -50,3 +50,6 @@ nosetests.xml
 # uv
 .venv
 uv.lock
+
+# Local container test runs (see run_tests_containers.sh)
+work/

.gitignore

@@ -3,7 +3,7 @@
 [![GitHub repository](https://img.shields.io/badge/GitHub-Repository-black?logo=github)](https://github.com/tcalmant/python-javaobj/)
 [![Latest Version](https://img.shields.io/pypi/v/javaobj-py3.svg)](https://pypi.python.org/pypi/javaobj-py3/)
 [![License](https://img.shields.io/pypi/l/javaobj-py3.svg)](https://pypi.python.org/pypi/javaobj-py3/)
-[![CI Build](https://github.com/tcalmant/python-javaobj/actions/workflows/build-24.04.yml/badge.svg?branch=v3)](https://github.com/tcalmant/python-javaobj/actions/workflows/build-24.04.yml)
+[![CI Build](https://github.com/tcalmant/python-javaobj/actions/workflows/ci-build.yml/badge.svg?branch=master)](https://github.com/tcalmant/python-javaobj/actions/workflows/ci-build.yml)
 [![Coveralls status](https://coveralls.io/repos/tcalmant/python-javaobj/badge.svg?branch=master)](https://coveralls.io/r/tcalmant/python-javaobj?branch=master)
 
 *python-javaobj* is a python library that provides functions for reading and
@@ -122,7 +122,9 @@ You can find a sample usage in the *Custom Transformer* section in this file.
 * Python >= 3.12 for `v3`
 * `enum34` and `typing` when using Python <= 3.4 (installable with `pip`)
 * Maven 2+ (for building test data of serialized objects.
-  You can skip it if you do not plan to run `tests.py`)
+  The fixtures are already committed under `tests/`, so you only need Maven
+  to regenerate them; skip it otherwise, or set `JAVAOBJ_NO_MAVEN=1` to
+  disable the regeneration when running the test suite)
 
 ## Usage (V1 implementation)
 

README.md

@@ -15,8 +15,6 @@
 :status: Alpha
 """
 
-from __future__ import unicode_literals
-
 import sys
 
 # Module version
@@ -26,6 +24,11 @@
 # Documentation strings format
 __docformat__ = "restructuredtext en"
 
+# Note: this module deliberately does NOT `from __future__ import
+# unicode_literals`. UnicodeDecodeError requires its encoding-name,
+# object and reason arguments to be the *native* str type; on Python 2
+# that means plain bytes-string literals, not unicode ones.
+
 # Encoding name: not cesu-8, which uses a different zero-byte
 NAME = "mutf8"
 
@@ -170,14 +173,19 @@ def decoder(data):
 
     def next_byte(_it, start, count):
         try:
-            return next(_it)[1]
+            return byte_to_int(next(_it)[1])
         except StopIteration:
             raise UnicodeDecodeError(
                 NAME, data, start, start + count, "incomplete byte sequence"
             )
 
     it = iter(enumerate(data))
-    for i, d in it:
+    for i, raw_d in it:
+        # Iterating a Python 3 bytes object yields ints, but iterating a
+        # Python 2 str yields single-character strings: normalize here so
+        # the bitwise logic below works the same on both, while `data`
+        # itself stays a real bytes-like object for UnicodeDecodeError.
+        d = byte_to_int(raw_d)
         if d == 0x00:  # 00000000
             raise UnicodeDecodeError(
                 NAME, data, i, i + 1, "embedded zero-byte not allowed"
@@ -196,6 +204,11 @@ def next_byte(_it, start, count):
                         for i1, dm in enumerate(DECODE_MAP[6]):
                             d1 = next_byte(it, i, i1 + 1)
                             value = dm.apply(d1, value, data, i, i1 + 1)
+                        # The 6 bytes reconstruct the supplementary
+                        # character's 20-bit offset from the surrogate
+                        # pair; add back the base to get the real
+                        # code point (U+10000..U+10FFFF).
+                        value += 0x10000
                     else:  # 1110xxxx
                         value = d & 0x0F
                         for i1, dm in enumerate(DECODE_MAP[3]):
@@ -228,7 +241,11 @@ def decode_modified_utf8(data, errors="strict"):
     :raises UnicodeDecodeError: sequence is invalid.
     """
     value, length = "", 0
-    it = iter(decoder(byte_to_int(d) for d in data))
+    # decoder() normalizes each item internally (via byte_to_int) as it
+    # iterates, so the original bytes-like `data` can be passed directly;
+    # it also needs to stay a real bytes-like object here, since
+    # UnicodeDecodeError requires one for the errors decoder() raises.
+    it = iter(decoder(data))
     while True:
         try:
             value += next(it)
@@ -242,7 +259,10 @@ def decode_modified_utf8(data, errors="strict"):
             if errors == "ignore":
                 pass
             elif errors == "replace":
-                value += "\uFFFD"
+                # Explicit u-prefix: without `unicode_literals` active in
+                # this module, a plain literal would not interpret \u as
+                # an escape sequence on Python 2.
+                value += u"\uFFFD"
                 length += 1
     return value, length
 

javaobj/modifiedutf8.py

@@ -61,7 +61,7 @@ def __init__(self, unmarshaller):
             JavaObject.__init__(self)
 
         def __hash__(self):
-            return list.__hash__(self)
+            return object.__hash__(self)
 
         def __extra_loading__(self, unmarshaller, ident=0):
             # type: (JavaObjectUnmarshaller, int) -> None
@@ -115,7 +115,7 @@ def __init__(self, unmarshaller):
             JavaObject.__init__(self)
 
         def __hash__(self):
-            return dict.__hash__(self)
+            return object.__hash__(self)
 
         def __extra_loading__(self, unmarshaller, ident=0):
             # type: (JavaObjectUnmarshaller, int) -> None
@@ -171,7 +171,7 @@ def __init__(self, unmarshaller):
             JavaObject.__init__(self)
 
         def __hash__(self):
-            return set.__hash__(self)
+            return object.__hash__(self)
 
         def __extra_loading__(self, unmarshaller, ident=0):
             # type: (JavaObjectUnmarshaller, int) -> None

javaobj/v1/transformers.py

@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Runs the python-javaobj test suite for the current Python interpreter.
+#
+# Used both inside the per-version containers driven by
+# run_tests_containers.sh, and directly in CI on the runner. javaobj.v3
+# requires Python 3.12+, so it (and its tests) are skipped below that version,
+# matching the compatibility table in README.md.
+#
+
+set -uo pipefail
+
+echo "Installing dependencies..."
+python -m pip install --upgrade pip
+pip install pytest coverage || exit 1
+if [ -f requirements.txt ]; then
+    pip install -r requirements.txt || exit 1
+fi
+
+python_supports_v3() {
+    python -c 'import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)'
+}
+
+if python_supports_v3
+then
+    echo "Python 3.12+: running the full suite (v1, v2, v3)..."
+    coverage run -m pytest
+    rc=$?
+else
+    echo "Python < 3.12: javaobj.v3 is unsupported, skipping it and its tests..."
+    coverage run --omit='javaobj/v3/*,tests/test_v3.py' -m pytest --ignore=tests/test_v3.py
+    rc=$?
+fi
+
+coverage report
+
+exit "$rc"