From c7d50975f7ea164429f8d8ca1b7a39f84d2c8e4f Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Wed, 2 Sep 2026 17:14:02 -0500 Subject: [PATCH 1/4] Drop setup.py in favour of pyproject.toml; bump to 0.2.0 All packaging metadata now lives in pyproject.toml: version, readme, license (SPDX), authors, classifiers, URLs, the console scripts and the Cython extension. setup.py is removed. Two parts of the extension build cannot be expressed statically in pyproject.toml -- NumPy's include directory is only known once NumPy is importable, and [tool.setuptools] ext-modules cannot express define-macros two-tuples. Both are handled by a small build_ext subclass in _cmac_build.py, wired up via [tool.setuptools.cmdclass]. MANIFEST.in ships that helper in the sdist so source installs still build. packages.find sets namespaces = false to match the previous find_packages() behaviour, keeping the un-packaged cmac/tests out of the distribution. The dead write_version_py/git_version helpers in setup.py were never called (ISRELEASED was undefined) and are dropped with it, so the version is now simply static. Verified: wheel build, sdist build, sdist -> wheel under full build isolation, editable install, and pytest cmac/tests (58 passed). Co-Authored-By: Claude Opus 5 (1M context) --- MANIFEST.in | 4 ++ _cmac_build.py | 25 +++++++++ pyproject.toml | 55 ++++++++++++++++++-- setup.py | 138 ------------------------------------------------- 4 files changed, 79 insertions(+), 143 deletions(-) create mode 100644 MANIFEST.in create mode 100644 _cmac_build.py delete mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..f06b653 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +# _cmac_build.py is a build-time helper referenced by +# [tool.setuptools.cmdclass]; it is not part of any package, so it has to be +# added to the sdist explicitly or building from the sdist fails. +include _cmac_build.py diff --git a/_cmac_build.py b/_cmac_build.py new file mode 100644 index 0000000..a5886e7 --- /dev/null +++ b/_cmac_build.py @@ -0,0 +1,25 @@ +"""Build-time customization for the ``cmac.calc_kdp_ray_fir`` Cython extension. + +Two parts of the extension build cannot be expressed statically in +``pyproject.toml``: NumPy's C header directory is only discoverable once +NumPy is importable, and ``[tool.setuptools] ext-modules`` cannot express +``define-macros`` two-tuples. Both are applied here and wired up through +``[tool.setuptools.cmdclass]``, so no ``setup.py`` is needed. +""" + +from setuptools.command.build_ext import build_ext as _build_ext + +# Compile against the stable NumPy C API rather than the deprecated one. +NUMPY_API_MACRO = ("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION") + + +class build_ext(_build_ext): + """``build_ext`` that resolves the NumPy include directory at build time.""" + + def finalize_options(self): + super().finalize_options() + import numpy + + self.include_dirs.append(numpy.get_include()) + for ext in self.distribution.ext_modules or []: + ext.define_macros.append(NUMPY_API_MACRO) diff --git a/pyproject.toml b/pyproject.toml index abaa296..b0d7ed5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,47 @@ [build-system] requires = [ - "setuptools>=45", + # >=77 for SPDX `license` / `license-files` support. + "setuptools>=77", "wheel", "cython", - "oldest-supported-numpy; python_version<'3.9'", - "numpy>=2.0; python_version>='3.9'", + # Building against NumPy 2 yields an extension that also runs on NumPy 1. + "numpy>=2.0", ] build-backend = "setuptools.build_meta" [project] name = "cmac" +version = "0.2.0" description = "Corrected Moments in Antenna Coordinates" -dynamic = ["readme", "version"] +readme = "README.rst" +requires-python = ">=3.10" +license = "BSD-3-Clause" +license-files = ["LICENSE", "LICENSE_GPL.txt"] +authors = [ + { name = "Scott Collis" }, + { name = "Zachary Sherman" }, + { name = "Robert Jackson" }, +] +maintainers = [ + { name = "Data Informatics and Geophysical Retrievals (DIGR)" }, +] +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Atmospheric Science", +] + +[project.urls] +Homepage = "https://github.com/ARM-Development/cmac" +Source = "https://github.com/ARM-Development/cmac" +Documentation = "https://www.arm.gov/data/data-sources/cmac-69" [project.optional-dependencies] # Install with `pip install -e .[test]` to get the testing extras. @@ -26,7 +56,22 @@ test = [ "act-atmos"] [tool.setuptools] -# Explicitly register the extension modules to be compiled +script-files = [ + "scripts/cmac", + "scripts/cmac_animation", + "scripts/cmac_dask", +] +# Explicitly register the extension modules to be compiled. The NumPy include +# directory and the NumPy API macro are added by _cmac_build.build_ext below. ext-modules = [ { name = "cmac.calc_kdp_ray_fir", sources = ["cmac/calc_kdp_ray_fir.pyx"] } ] + +[tool.setuptools.packages.find] +# namespaces = false matches the old find_packages() behaviour: cmac/tests has +# no __init__.py and is deliberately left out of the distribution. +include = ["cmac*"] +namespaces = false + +[tool.setuptools.cmdclass] +build_ext = "_cmac_build.build_ext" diff --git a/setup.py b/setup.py deleted file mode 100644 index 231950d..0000000 --- a/setup.py +++ /dev/null @@ -1,138 +0,0 @@ -""" CMAC Corrected Precipitation Radar Moments in Antenna Coordinates - -Using fuzzy logic, scipy, and more to identify gates as rain, melting, -snow, no clutter, and second trip. Many fields such as reflectivity and -coorelation coefficient are used, but also SNR and sounding data is used. -More information can be found at https://www.arm.gov/data/data-sources/cmac-69 - -""" - - -import os -import subprocess -from setuptools import setup, find_packages, Extension - -import numpy as np -from Cython.Build import cythonize - -DOCLINES = __doc__.split("\n") - -CLASSIFIERS = """\ -Development Status :: 2 - Pre-Alpha -Intended Audience :: Science/Research -Intended Audience :: Developers -License :: OSI Approved :: BSD License -Programming Language :: Python -Programming Language :: Python :: 3.6 -Topic :: Scientific/Engineering -Topic :: Scientific/Engineering :: Atmospheric Science -Operating System :: POSIX :: Linux -""" - -NAME = 'cmac' -AUTHOR = 'Scott Collis, Zachary Sherman, Robert Jackson' -MAINTAINER = 'Data Informatics and Geophysical Retrievals (DIGR)' -DESCRIPTION = DOCLINES[0] -LONG_DESCRIPTION = "\n".join(DOCLINES[2:]) -URL = 'https://github.com/EVS-ATMOS/cmac2.0' -LICENSE = 'BSD' -CLASSIFIERS = filter(None, CLASSIFIERS.split('\n')) -MAJOR = 0 -MINOR = 1 -MICRO = 0 -VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) - - -# Return the git revision as a string -def git_version(): - def _minimal_ext_cmd(cmd): - # construct minimal environment - env = {} - for k in ['SYSTEMROOT', 'PATH']: - v = os.environ.get(k) - if v is not None: - env[k] = v - # LANGUAGE is used on win32 - env['LANGUAGE'] = 'C' - env['LANG'] = 'C' - env['LC_ALL'] = 'C' - out = subprocess.Popen( - cmd, stdout=subprocess.PIPE, env=env).communicate()[0] - return out - - try: - out = _minimal_ext_cmd(['git', 'rev-parse', 'HEAD']) - GIT_REVISION = out.strip().decode('ascii') - except OSError: - GIT_REVISION = "Unknown" - - return GIT_REVISION - - -def write_version_py(filename='cmac/version.py'): - cnt = """ -# THIS FILE IS GENERATED FROM PYART SETUP.PY -short_version = '%(version)s' -version = '%(version)s' -full_version = '%(full_version)s' -git_revision = '%(git_revision)s' -release = %(isrelease)s - -if not release: - version = full_version -""" - # Adding the git rev number needs to be done inside write_version_py(), - # otherwise the import of cmac.version messes up the build under Python 3. - FULLVERSION = VERSION - if os.path.exists('.git'): - GIT_REVISION = git_version() - elif os.path.exists('cmac/version.py'): - # must be a source distribution, use existing version file - try: - from cmac.version import git_revision as GIT_REVISION - except ImportError: - raise ImportError("Unable to import git_revision. Try removing " - "cmac/version.py and the build directory " - "before building.") - else: - GIT_REVISION = "Unknown" - - if not ISRELEASED: - FULLVERSION += '.dev+' + GIT_REVISION[:7] - - a = open(filename, 'w') - try: - a.write(cnt % {'version': VERSION, - 'full_version': FULLVERSION, - 'git_revision': GIT_REVISION, - 'isrelease': str(ISRELEASED)}) - finally: - a.close() - - -extensions = [ - Extension( - 'cmac.calc_kdp_ray_fir', - sources=['cmac/calc_kdp_ray_fir.pyx'], - include_dirs=[np.get_include()], - define_macros=[('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')], - ), -] - - -setup( - name=NAME, - version=VERSION, - description=DESCRIPTION, - long_description=LONG_DESCRIPTION, - url=URL, - author=AUTHOR, - maintainer=MAINTAINER, - license=LICENSE, - classifiers=CLASSIFIERS, - packages=find_packages(), - ext_modules=cythonize(extensions), - scripts=['scripts/cmac', - 'scripts/cmac_animation', - 'scripts/cmac_dask'] -) From 8a28121ab790d026f9471e5ae8af1eaa43fb274e Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Wed, 2 Sep 2026 17:40:36 -0500 Subject: [PATCH 2/4] Use the conda-forge compiler to build the Cython extension in CI conda-forge's Python bakes the flags of the GCC it was built with into sysconfig's CFLAGS. Compiling cmac.calc_kdp_ray_fir with the runner's older system gcc failed on flags it does not recognise: gcc: error: unrecognized command-line option '-partition=none'; did you mean '-flto-partition=none'? Add c-compiler to the CI environment so a matching conda-forge gcc is installed and exported as CC on environment activation. Co-Authored-By: Claude Opus 5 (1M context) --- continuous_integration/environment-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/continuous_integration/environment-ci.yml b/continuous_integration/environment-ci.yml index 75cf8f4..1ada781 100644 --- a/continuous_integration/environment-ci.yml +++ b/continuous_integration/environment-ci.yml @@ -3,6 +3,12 @@ channels: - conda-forge - defaults dependencies: + # conda-forge's Python bakes the flags of the GCC it was built with into + # sysconfig's CFLAGS. Compiling the Cython extension with the runner's older + # system gcc then fails on flags it does not know (e.g. `-partition=none`). + # c-compiler pulls in the matching conda-forge gcc and sets CC/CFLAGS on + # environment activation, so the extension builds against its own toolchain. + - c-compiler - arm_pyart - cartopy - cmweather From beb77c7a93bbb7c8f501db73ac65d152d1c59396 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Wed, 2 Sep 2026 17:47:34 -0500 Subject: [PATCH 3/4] Drop compiler flags the building compiler does not support A conda-forge interpreter records the flags of whichever GCC built it in sysconfig's CFLAGS, and customize_compiler copies those verbatim into every command line. Python 3.12/3.13 on conda-forge now carry -partition=none from GCC 16, which neither the runner's system gcc nor the conda-forge c-compiler pin (gcc 15.3.0) accepts: gcc: error: unrecognized command-line option '-partition=none'; did you mean '-flto-partition=none'? Adding c-compiler to the CI environment did not help, since its pinned gcc trails the one that built the interpreter; revert that and instead probe the compiler in build_ext and strip the options it rejects. These are optimization and LTO tuning flags, so dropping them changes nothing semantically, and this also fixes source installs outside CI. Co-Authored-By: Claude Opus 5 (1M context) --- _cmac_build.py | 104 ++++++++++++++++++++-- continuous_integration/environment-ci.yml | 6 -- 2 files changed, 99 insertions(+), 11 deletions(-) diff --git a/_cmac_build.py b/_cmac_build.py index a5886e7..e3d9725 100644 --- a/_cmac_build.py +++ b/_cmac_build.py @@ -1,20 +1,103 @@ """Build-time customization for the ``cmac.calc_kdp_ray_fir`` Cython extension. -Two parts of the extension build cannot be expressed statically in +Three parts of the extension build cannot be expressed statically in ``pyproject.toml``: NumPy's C header directory is only discoverable once -NumPy is importable, and ``[tool.setuptools] ext-modules`` cannot express -``define-macros`` two-tuples. Both are applied here and wired up through -``[tool.setuptools.cmdclass]``, so no ``setup.py`` is needed. +NumPy is importable, ``[tool.setuptools] ext-modules`` cannot express +``define-macros`` two-tuples, and the interpreter's own ``CFLAGS`` may name +options the available compiler does not accept. All three are applied here +and wired up through ``[tool.setuptools.cmdclass]``, so no ``setup.py`` is +needed. """ +import os +import re +import subprocess +import tempfile + from setuptools.command.build_ext import build_ext as _build_ext # Compile against the stable NumPy C API rather than the deprecated one. NUMPY_API_MACRO = ("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION") +# gcc: "unrecognized command-line option '-partition=none'" +# clang: "unknown argument: '-partition=none'" +UNSUPPORTED_FLAG = re.compile( + r"""(?:unrecognized\s+(?:command[-\s]line\s+)?option|unknown\s+argument:?)""" + r"""\s*['"]?(-[^\s'"();]+)""", + re.IGNORECASE, +) + +# Every compiler/linker command line distutils may hand us. Which of these +# exist varies with the setuptools version, so they are probed by name. +FLAG_ATTRS = ( + "compiler", + "compiler_so", + "compiler_so_cxx", + "compiler_cxx", + "linker_so", + "linker_so_cxx", + "linker_exe", +) + + +def _rejected_flags(command, source): + """Return the flags in ``command`` that the compiler refuses outright. + + Returns an empty set when the trial compile succeeds, and also when it + fails for any other reason -- this is a best-effort cleanup, so a real + build error is left for the real build to report. + """ + probe = subprocess.run( + list(command) + ["-c", source, "-o", os.devnull], + capture_output=True, + text=True, + ) + if probe.returncode == 0: + return set() + return set(UNSUPPORTED_FLAG.findall(probe.stderr)) + + +def drop_unsupported_flags(compiler): + """Strip options the compiler rejects from ``compiler``'s command lines. + + A conda-forge interpreter records the flags of whichever GCC built it in + ``sysconfig``'s ``CFLAGS``, and ``customize_compiler`` copies those into + every command line verbatim. When the compiler doing the building is not + that same GCC -- an older system gcc, or a conda toolchain pinned a + release behind -- the build dies on an option it has never heard of, e.g. + ``-partition=none`` from GCC 16. Dropping those flags costs nothing: they + are optimization and LTO tuning, not semantics. + """ + if not getattr(compiler, "compiler_so", None): + return set() + + dropped = set() + with tempfile.TemporaryDirectory() as tmpdir: + source = os.path.join(tmpdir, "flag_probe.c") + with open(source, "w") as handle: + handle.write("int main(void) { return 0; }\n") + + # Each pass can only surface the flags the compiler reaches before + # giving up, so re-probe until it stops complaining. The bound just + # guarantees termination if a flag somehow survives removal. + for _ in range(8): + rejected = _rejected_flags(compiler.compiler_so, source) - dropped + if not rejected: + break + dropped |= rejected + for attr in FLAG_ATTRS: + command = getattr(compiler, attr, None) + if command: + setattr( + compiler, + attr, + [arg for arg in command if arg not in rejected], + ) + return dropped + class build_ext(_build_ext): - """``build_ext`` that resolves the NumPy include directory at build time.""" + """``build_ext`` that resolves NumPy's headers and sanitizes CFLAGS.""" def finalize_options(self): super().finalize_options() @@ -23,3 +106,14 @@ def finalize_options(self): self.include_dirs.append(numpy.get_include()) for ext in self.distribution.ext_modules or []: ext.define_macros.append(NUMPY_API_MACRO) + + def build_extensions(self): + # Runs after distutils has built self.compiler from sysconfig. + dropped = drop_unsupported_flags(self.compiler) + if dropped: + self.announce( + "dropping compiler flags not supported by this compiler: " + + " ".join(sorted(dropped)), + level=2, + ) + super().build_extensions() diff --git a/continuous_integration/environment-ci.yml b/continuous_integration/environment-ci.yml index 1ada781..75cf8f4 100644 --- a/continuous_integration/environment-ci.yml +++ b/continuous_integration/environment-ci.yml @@ -3,12 +3,6 @@ channels: - conda-forge - defaults dependencies: - # conda-forge's Python bakes the flags of the GCC it was built with into - # sysconfig's CFLAGS. Compiling the Cython extension with the runner's older - # system gcc then fails on flags it does not know (e.g. `-partition=none`). - # c-compiler pulls in the matching conda-forge gcc and sets CC/CFLAGS on - # environment activation, so the extension builds against its own toolchain. - - c-compiler - arm_pyart - cartopy - cmweather From e5c52a9f6bc341265e29fad6359d16a6d856939e Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Wed, 2 Sep 2026 17:51:45 -0500 Subject: [PATCH 4/4] Match GCC's non-ASCII quoting when parsing rejected flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit's probe never fired in CI: under a UTF-8 locale GCC quotes the offending option with U+2018/U+2019 ("unrecognized command-line option ‘-partition=none’"), while the pattern assumed the ASCII quoting clang had produced locally, so nothing was stripped and the build failed exactly as before. Run the probe under LC_ALL=C so diagnostics come back in ASCII, and match the quoting loosely as a backstop in case that is ignored. Co-Authored-By: Claude Opus 5 (1M context) --- _cmac_build.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/_cmac_build.py b/_cmac_build.py index e3d9725..217e12e 100644 --- a/_cmac_build.py +++ b/_cmac_build.py @@ -21,12 +21,21 @@ # gcc: "unrecognized command-line option '-partition=none'" # clang: "unknown argument: '-partition=none'" +# GCC quotes with U+2018/U+2019 under a UTF-8 locale, so the quoting around +# the flag is matched loosely rather than assumed to be ASCII. +QUOTES = "\"'`\u2018\u2019\u201c\u201d" UNSUPPORTED_FLAG = re.compile( - r"""(?:unrecognized\s+(?:command[-\s]line\s+)?option|unknown\s+argument:?)""" - r"""\s*['"]?(-[^\s'"();]+)""", + r"(?:unrecognized\s+(?:command[-\s]line\s+)?option" + r"|unknown\s+argument:?)" + r"\s*[" + QUOTES + r"]*\s*" + r"(-[^\s,;()" + QUOTES + r"]+)", re.IGNORECASE, ) +# Ask the compiler for ASCII diagnostics so the pattern above has the easiest +# possible job; the loose quoting stays as a backstop if this is ignored. +C_LOCALE_ENV = dict(os.environ, LC_ALL="C", LANG="C") + # Every compiler/linker command line distutils may hand us. Which of these # exist varies with the setuptools version, so they are probed by name. FLAG_ATTRS = ( @@ -51,6 +60,7 @@ def _rejected_flags(command, source): list(command) + ["-c", source, "-o", os.devnull], capture_output=True, text=True, + env=C_LOCALE_ENV, ) if probe.returncode == 0: return set()