diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 33efd207..4de28a84 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,6 +3,8 @@ on: push: branches: - master + tags: + - 'v*' pull_request: types: - opened @@ -13,10 +15,14 @@ jobs: build-and-test: strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + + include: # Used to choose which version of Python to use for docs and for sdist + - python-version: "3.12" + canonical: true runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: git setup @@ -42,12 +48,18 @@ jobs: # with pip to make sure it works. run: | source "${HOME}/conda/etc/profile.d/conda.sh" - conda create -p ./cython-env -y "cython>3.0" python=${{ matrix.python-version }} gxx zlib setuptools --channel conda-forge + conda create -p ./cython-env -y "cython>3.0" python=${{ matrix.python-version }} gxx zlib setuptools xz --channel conda-forge conda activate ./cython-env python setup.py clean cythonize sdist (cd dist && pip install pybedtools-*.tar.gz && cd $TMPDIR && python -c 'import pybedtools; print(pybedtools.__file__)') conda deactivate + - name: upload sdist + if: matrix.canonical + uses: actions/upload-artifact@v7 + with: + name: sdist + path: dist/pybedtools-*.tar.gz - name: conda env and install locally # Set up conda and install pybedtools into that env @@ -106,7 +118,7 @@ jobs: - name: build-docs - if: ${{ (matrix.python-version == 3.10) }} + if: matrix.canonical # Build docs and commit to gh-pages branch. Note that no push happens # unless we're on the master branch run: | @@ -117,17 +129,29 @@ jobs: cd /tmp/pybedtools-uncompressed/pybedtools-* (cd docs && make html) - git clone \ - --single-branch \ - --branch gh-pages "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY" \ - /tmp/docs + REPO_URL="https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY" + + # The gh-pages branch may not exist upstream (e.g. in a fork that has + # never published docs); if so, create a new orphan branch to work in. + if git ls-remote --exit-code --heads "$REPO_URL" gh-pages > /dev/null 2>&1; then + git clone \ + --single-branch \ + --branch gh-pages "$REPO_URL" \ + /tmp/docs + else + echo "gh-pages branch not found upstream; creating an orphan branch locally" + git clone "$REPO_URL" /tmp/docs + git -C /tmp/docs checkout --orphan gh-pages + git -C /tmp/docs rm -rf --quiet . || true + fi rm -rf /tmp/docs/* cp -r docs/build/html/* /tmp/docs touch /tmp/docs/.nojekyll cd /tmp/docs git add . - if git diff --cached --quiet; then + # with --verify HEAD, if there's no HEAD then we should commit + if git rev-parse --verify HEAD > /dev/null 2>&1 && git diff --cached --quiet; then echo "no changes, nothing to commit" else git commit -m 'update docs' @@ -137,7 +161,8 @@ jobs: - name: docs artifact # Upload built docs as an artifact for inspection, even on PRs - uses: actions/upload-artifact@v4 + if: matrix.canonical + uses: actions/upload-artifact@v7 with: name: docs path: /tmp/docs @@ -145,8 +170,38 @@ jobs: - name: push docs to gh-pages branch # Push docs to gh-pages if this test is running on master branch - if: ${{ (github.ref == 'refs/heads/master') && (matrix.python-version == 3.10) }} + if: (github.ref == 'refs/heads/master') && matrix.canonical run: | cd /tmp/docs git push "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY" gh-pages cd $WORKDIR + + + build-wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + cibw_archs: "x86_64" + - os: ubuntu-24.04-arm + cibw_archs: "aarch64" + - os: macos-latest + cibw_archs: "arm64 x86_64" + + steps: + - uses: actions/checkout@v7 + + - name: Build wheels + uses: pypa/cibuildwheel@v4.2.0 + env: + CIBW_ARCHS: ${{ matrix.cibw_archs }} + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" + CIBW_SKIP: "*-win*" + CIBW_BEFORE_BUILD: "pip install 'cython>3.0' setuptools && python setup.py cythonize" + + - uses: actions/upload-artifact@v7 + with: + name: wheels-${{ matrix.os }} + path: ./wheelhouse/*.whl diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 00000000..0706ffb3 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,54 @@ +# Releases + +As of v0.12.1, wheels are built as part of the GitHub Actions workflow. Rather +than tie these in to an automated push-to-PyPI process, for now publishing to +PyPI continues as a manual process -- but now with wheels in addition to the +sdist. + +First, merge to master and wait for CI to finish. + +Get the run ID from the merge's GitHub Actions run. Use that, plus the `gh` +command-line tool, to download the artifacts. + +```bash +RUN_ID="" +rm -r staging && mkdir -p staging +rm -r dist && mkdir -p dist + +# Download all wheels and the sdist, store 'em in staging/. +# We'll have subdirectories for each arch/os +gh run download $RUN_ID -p 'wheels-*' -p sdist -D staging + +# Flatten nested wheels & sdist into the dist/ dir +mkdir dist && find staging -type f \( -name '*.whl' -o -name '*.tar.gz' \) -exec mv {} dist/ \; +``` + +Tag the release. + +```bash +TAG="" +git tag -s $TAG && git push origin $TAG +``` + +Check the dist, run a local test install, then upload to PyPI. + +```bash +ls dist/ + +# x-rst warnings OK +twine check dist/* + +# Test local on linux +python -m venv /tmp/t && /tmp/t/bin/pip install dist/pybedtools-$(echo $TAG | sed "s/v//")-cp312-*manylinux*.whl + +# Or mac +python -m venv /tmp/t && /tmp/t/bin/pip install dist/pybedtools-$(echo $TAG | sed "s/v//")-cp312-*macosx*_arm64.whl + +# Test PyPI +twine upload -r testpypi dist/* + +# Prod PyPI +twine upload dist/* +``` + +Bioconda builds should pick it up in an hour or so after pushing to PyPI. diff --git a/pybedtools/bedtool.py b/pybedtools/bedtool.py index fb34726a..6accc569 100644 --- a/pybedtools/bedtool.py +++ b/pybedtools/bedtool.py @@ -750,7 +750,11 @@ def tabix_intervals(self, interval_or_string: Interval | str, check_coordinates: # tabix expects 1-based coords, but BEDTools works with # zero-based. pybedtools and pysam also work with zero-based. So we can # pass zero-based directly to the pysam tabix interface. - tbx = pysam.TabixFile(self.fn) + try: + tbx = pysam.TabixFile(self.fn) + except OSError: + # if the file is indexed using csi, we need to specify the path for index + tbx = pysam.TabixFile(self.fn, index=self.fn+".csi") # If an interval is passed, use its coordinates directly if isinstance(interval_or_string, Interval): @@ -799,10 +803,15 @@ def tabix_contigs(self): "-- please use the .tabix() method" ) - tbx = pysam.TabixFile(self.fn) + try: + tbx = pysam.TabixFile(self.fn) + except OSError: + # if the file is indexed using csi, we need to specify the path for index + tbx = pysam.TabixFile(self.fn, index=self.fn+".csi") return tbx.contigs - def tabix(self, in_place: bool = True, force: bool = False, is_sorted: bool = False) -> BedTool: + + def tabix(self, in_place: bool = True, force: bool = False, is_sorted: bool = False, use_csi: bool = False) -> BedTool: """ Prepare a BedTool for use with Tabix. @@ -823,6 +832,10 @@ def tabix(self, in_place: bool = True, force: bool = False, is_sorted: bool = Fa is_sorted : bool If True (default is False), then assume the file is already sorted so that BedTool.bgzip() doesn't have to do that work. + + use_csi : bool + If True (default is False), then generate a csi instead of tbi index. + This can be useful when working with chromosomes larger than 512 Mbp, such as barley """ # Return quickly if nothing to do if self._tabixed() and not force: @@ -831,18 +844,19 @@ def tabix(self, in_place: bool = True, force: bool = False, is_sorted: bool = Fa # Make sure it's BGZIPed fn = self.bgzip(in_place=in_place, force=force, is_sorted=is_sorted) if self.file_type is not None and self.file_type not in ["bam", "empty"]: - pysam.tabix_index(fn, force=force, preset=self.file_type) # type: ignore + pysam.tabix_index(fn, force=force, preset=self.file_type, csi=use_csi) # type: ignore + return BedTool(fn) def _tabixed(self): """ Verifies that we're working with a tabixed file: a string filename - pointing to a BGZIPed file with a .tbi file in the same dir. + pointing to a BGZIPed file with a .tbi or .csi file in the same dir. """ if ( isinstance(self.fn, str) and isBGZIP(self.fn) - and os.path.exists(self.fn + ".tbi") + and (os.path.exists(self.fn + ".tbi") or os.path.exists(self.fn + ".csi")) ): return True diff --git a/pybedtools/test/test_1.py b/pybedtools/test/test_1.py index 044b495b..aa5ff461 100644 --- a/pybedtools/test/test_1.py +++ b/pybedtools/test/test_1.py @@ -128,6 +128,38 @@ def test_tuple_creation(): assert x[0]["ID"] == "gene1" +def test_tabix_csi(): + for idx_type in ("tbi", "csi"): + try: + a = pybedtools.example_bedtool("a.bed") + t = a.tabix(force=True, use_csi=True if idx_type == "csi" else False) + assert t._tabixed() + results = t.tabix_intervals("chr1:99-200") + results = str(results) + print(results) + assert results == fix( + """ + chr1 1 100 feature1 0 + + chr1 100 200 feature2 0 + + chr1 150 500 feature3 0 -""" + ) + + assert str(t.tabix_intervals(a[2])) == fix( + """ + chr1 100 200 feature2 0 + + chr1 150 500 feature3 0 -""" + ) + + finally: + # clean up + fns = [ + pybedtools.example_filename("a.bed.gz"), + pybedtools.example_filename("a.bed.gz." + idx_type), + ] + for fn in fns: + if os.path.exists(fn): + os.unlink(fn) + def test_tabix(tmp_path: Path) -> None: shutil.copy(os.path.join(filenames.data_dir(), "a.bed"), tmp_path) a = pybedtools.BedTool(tmp_path / "a.bed") @@ -160,6 +192,12 @@ def test_tabix_intervals(): assert len(a.tabix_intervals("chr1")) == 1 +def test_tabix_contigs_csi(): + a = pybedtools.example_bedtool("a.bed") + a = a.tabix(force=True, use_csi=True) + assert a.tabix_contigs() == ["chr1"] + + # ---------------------------------------------------------------------------- # Streaming and non-file BedTool tests # ---------------------------------------------------------------------------- diff --git a/pybedtools/test/test_issues.py b/pybedtools/test/test_issues.py index 810e4100..22a4901a 100644 --- a/pybedtools/test/test_issues.py +++ b/pybedtools/test/test_issues.py @@ -8,6 +8,7 @@ from pathlib import Path import pytest import psutil +import gc from pybedtools import filenames @@ -61,14 +62,29 @@ def test_issue_81(): def test_issue_118(): p = psutil.Process(os.getpid()) - start_fds = p.num_fds() a = pybedtools.example_bedtool("a.bed") b = pybedtools.example_bedtool("b.bed") + + # p.num_fds() counts anything running on the machine. Recently (Aug 2026), + # some GitHub Actions tests failed the start == stop fds because we had + # *fewer* fds. This could occur from things outside of pytest that we can't + # control cleaning up fds. The big thing we're checking here is that we + # don't leak fds during this individual test. + # + # To make this a more accurate test, do a warm-up intersection field count, + # then garbage collect, then do the loop, then garbage collect again. + a.intersect(b).field_count() + gc.collect() + start_fds = p.num_fds() for i in range(100): c = a.intersect(b) c.field_count() + gc.collect() + stop_fds = p.num_fds() - assert start_fds == stop_fds + + # Could have had GC run + assert p.num_fds() <= start_fds def test_issue_131(): diff --git a/setup.py b/setup.py index bf234d24..e0e646b3 100644 --- a/setup.py +++ b/setup.py @@ -71,8 +71,8 @@ MAJ = 0 MIN = 12 -REV = 0 -VERSION = '%d.%d.%d' % (MAJ, MIN, REV) +REV = 1 +VERSION = f'{MAJ}.{MIN}.{REV}' class CleanCommand(Command):