From d01771b978bcb9967b33cbeb6ef56e802b67666d Mon Sep 17 00:00:00 2001 From: Daniel Alome Date: Sun, 4 Oct 2026 15:16:07 +0100 Subject: [PATCH 1/3] ADFA-4851 | Python-Tools: syntax highlighting and code completion for .py files Python-Tools implements LanguageExtension: it ships the tree-sitter Python grammar (0.23.6, built per ABI from the checksummed npm tarball) with its highlights query, and runs pylsp on the device for completion, pyflakes diagnostics and yapf formatting. pylsp and the Flask template's packages come from a bundled wheelhouse pinned in python-bundle.lock (URL + SHA-256), installed with pip --no-index, so neither needs a network connection. min_ide_version moves to 26.41 and native.code is requested for the grammar library. --- plugins/Python-Tools/.gitignore | 1 + plugins/Python-Tools/README.md | 13 +- plugins/Python-Tools/build.gradle.kts | 196 +++++++++++++++++- plugins/Python-Tools/python-bundle.lock | 31 +++ .../Python-Tools/src/main/AndroidManifest.xml | 4 +- .../src/main/assets/treesitter/python/LICENSE | 21 ++ .../assets/treesitter/python/highlights.scm | 137 ++++++++++++ .../Python-Tools/src/main/cpp/CMakeLists.txt | 11 + .../python/plugin/PythonToolsPlugin.kt | 175 +++++++++++++--- 9 files changed, 560 insertions(+), 29 deletions(-) create mode 100644 plugins/Python-Tools/python-bundle.lock create mode 100644 plugins/Python-Tools/src/main/assets/treesitter/python/LICENSE create mode 100644 plugins/Python-Tools/src/main/assets/treesitter/python/highlights.scm create mode 100644 plugins/Python-Tools/src/main/cpp/CMakeLists.txt diff --git a/plugins/Python-Tools/.gitignore b/plugins/Python-Tools/.gitignore index 30f6e1a1..ce96be66 100644 --- a/plugins/Python-Tools/.gitignore +++ b/plugins/Python-Tools/.gitignore @@ -17,3 +17,4 @@ local.properties # macOS .DS_Store +.cxx/ diff --git a/plugins/Python-Tools/README.md b/plugins/Python-Tools/README.md index e8531d08..a0ef6938 100644 --- a/plugins/Python-Tools/README.md +++ b/plugins/Python-Tools/README.md @@ -11,7 +11,18 @@ Two Python project templates: - **Python Flask App** — a Flask web app with routes, HTML templates, static files, and error handling. - **Python Starter** — a minimal Python project with a single entry point. -It also installs Python on-device (via Termux) the first time it is needed, and provides built-in run, install-dependencies, and test actions for Python projects. +It also installs Python on-device the first time it is needed, and provides built-in run, install-dependencies, and test actions for Python projects. + +`.py` files get syntax highlighting from a bundled tree-sitter Python grammar, and code completion, diagnostics (pyflakes), and formatting (yapf) from `pylsp`, which runs on the device. Both are contributed through the plugin API's `LanguageExtension`, so the plugin needs Code on the Go 26.41 or later. + +## Offline bundle + +Python 3.12 and pip ship in Code on the Go's Termux bootstrap, so the plugin only bundles what used to need a network connection: + +- **Wheelhouse** - python-lsp-server with pyflakes and yapf, Flask, gunicorn, and everything they depend on. On first use the plugin installs them with `pip install --no-index` from the bundled files. `ujson` and `markupsafe` have no Android wheels, so they ship as source and are compiled on the device by the bootstrap's `clang` (with `setuptools`, `setuptools-scm`, and `wheel` bundled for the build). +- **Project requirements** are installed automatically when a Python project opens, looking in the wheelhouse first, so the Flask template (Flask and gunicorn) needs no network. Other packages in a project's `requirements.txt` still download from PyPI. + +`python-bundle.lock` pins every file by URL and SHA-256. `downloadPythonBundle` fetches them at build time (resuming interrupted transfers) and refuses any file whose checksum does not match. To update, regenerate the lock from a `pip install --dry-run --report` run on a device. ## In-app help diff --git a/plugins/Python-Tools/build.gradle.kts b/plugins/Python-Tools/build.gradle.kts index bd85463f..d758c140 100644 --- a/plugins/Python-Tools/build.gradle.kts +++ b/plugins/Python-Tools/build.gradle.kts @@ -1,3 +1,10 @@ +import java.io.FileOutputStream +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URI +import java.security.MessageDigest +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { @@ -9,6 +16,161 @@ pluginBuilder { pluginName = "python-tools" } +data class GrammarSource( + val name: String, + val version: String, + val sha256: String, +) { + val url = "https://registry.npmjs.org/tree-sitter-$name/-/tree-sitter-$name-$version.tgz" +} + +val grammarSource = GrammarSource("python", "0.23.6", "aab5b860d93dbf84b37fed532f7dacc0468ede5387d0c74911d68f964287662a") +val grammarDir = layout.buildDirectory.dir("tree-sitter/${grammarSource.name}") + +val DOWNLOAD_ATTEMPTS = 40 + +fun sha256Of(file: File): String { + val digest = MessageDigest.getInstance("SHA-256") + file.inputStream().use { stream -> + val buffer = ByteArray(1 shl 16) + while (true) { + val read = stream.read(buffer) + if (read <= 0) break + digest.update(buffer, 0, read) + } + } + return digest.digest().joinToString("") { "%02x".format(it) } +} + +fun fetchVerified( + url: String, + sha256: String, + target: File, +) { + if (target.exists() && sha256Of(target) == sha256) return + target.parentFile.mkdirs() + target.delete() + logger.lifecycle("Downloading $url") + var attempt = 0 + while (true) { + attempt++ + val offset = target.length() + val connection = URI(url).toURL().openConnection() as HttpURLConnection + connection.connectTimeout = 30_000 + connection.readTimeout = 120_000 + if (offset > 0) connection.setRequestProperty("Range", "bytes=$offset-") + val interrupted = + try { + val append = offset > 0 && connection.responseCode == HttpURLConnection.HTTP_PARTIAL + connection.inputStream.use { input -> + FileOutputStream(target, append).use { output -> input.copyTo(output) } + } + null + } catch (e: IOException) { + e + } finally { + connection.disconnect() + } + if (interrupted == null && connection.contentLengthLong in listOf(-1L, target.length() - offset, target.length())) break + if (attempt >= DOWNLOAD_ATTEMPTS) { + target.delete() + throw GradleException("Failed to download $url after $attempt attempts: ${interrupted?.message}", interrupted) + } + logger.lifecycle("Resuming $url at ${target.length()} bytes (attempt ${attempt + 1})") + } + val actual = sha256Of(target) + if (actual != sha256) { + target.delete() + throw GradleException("Checksum mismatch for $url (expected $sha256, got $actual)") + } +} + +data class BundleEntry( + val directory: String, + val fileName: String, + val url: String, + val sha256: String, +) + +val pythonBundleLock = layout.projectDirectory.file("python-bundle.lock") +val pythonBundleCache = layout.buildDirectory.dir("python-bundle-cache") +val pythonBundleDir = layout.buildDirectory.dir("generated/pythonBundle") + +val downloadPythonBundle = + tasks.register("downloadPythonBundle") { + group = "setup" + description = "Download and verify the Python wheels bundled into the plugin" + + val lock = pythonBundleLock + val cacheDir = pythonBundleCache + val outputDir = pythonBundleDir + inputs.file(lock) + outputs.dir(outputDir) + + doLast { + val entries = + lock.asFile.readLines().filter { it.isNotBlank() }.map { line -> + val fields = line.split('\t') + if (fields.size != 4) throw GradleException("Malformed python-bundle.lock line: $line") + BundleEntry(fields[0], fields[1], fields[2], fields[3]) + } + val cache = cacheDir.get().asFile + entries.forEach { fetchVerified(it.url, it.sha256, File(cache, "${it.directory}/${it.fileName}")) } + + val zipFile = File(outputDir.get().asFile, "python/wheelhouse.zip") + zipFile.parentFile.deleteRecursively() + zipFile.parentFile.mkdirs() + ZipOutputStream(zipFile.outputStream()).use { zip -> + entries.filter { it.directory == "wheelhouse" }.sortedBy { it.fileName }.forEach { entry -> + zip.putNextEntry(ZipEntry(entry.fileName).apply { time = 0L }) + File(cache, "${entry.directory}/${entry.fileName}").inputStream().use { it.copyTo(zip) } + zip.closeEntry() + } + } + } + } + +val downloadGrammar = + tasks.register("downloadTreeSitterGrammar") { + group = "setup" + description = "Download, verify, and unpack the tree-sitter grammar compiled into the plugin" + + val source = grammarSource + val outputDir = grammarDir + val archive = layout.buildDirectory.file("tree-sitter/${source.name}-${source.version}.tgz") + outputs.dir(outputDir) + + doLast { + val archiveFile = archive.get().asFile + fetchVerified(source.url, source.sha256, archiveFile) + val target = outputDir.get().asFile + target.deleteRecursively() + project.copy { + from(project.tarTree(project.resources.gzip(archiveFile))) + include("package/src/**") + eachFile { path = path.removePrefix("package/") } + includeEmptyDirs = false + into(target) + } + } + } + +tasks.named("preBuild") { + dependsOn(downloadGrammar, downloadPythonBundle) +} + +tasks.matching { it.name.startsWith("configureCMake") || it.name.startsWith("buildCMake") }.configureEach { + dependsOn(downloadGrammar) +} + +downloadGrammar.configure { + mustRunAfter(tasks.named("clean")) +} + +downloadPythonBundle.configure { + mustRunAfter(tasks.named("clean")) +} + android { namespace = "com.appdevforall.python.plugin" compileSdk = 36 @@ -17,8 +179,38 @@ android { applicationId = "com.appdevforall.python.plugin" minSdk = 26 targetSdk = 36 - versionCode = 1 - versionName = "1.0.0" + versionCode = 3 + versionName = "1.2.0" + + ndk { + abiFilters += listOf("arm64-v8a", "armeabi-v7a") + } + + externalNativeBuild { + cmake { + arguments += "-DTREE_SITTER_GRAMMAR_NAME=${grammarSource.name}" + arguments += "-DTREE_SITTER_GRAMMAR_DIR=${grammarDir.get().asFile.absolutePath}" + arguments += "-DCMAKE_BUILD_TYPE=Release" + arguments += "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON" + } + } + } + + androidResources { + noCompress += listOf("zip") + } + + sourceSets { + named("main") { + assets.srcDir(pythonBundleDir.get().asFile) + } + } + + externalNativeBuild { + cmake { + path("src/main/cpp/CMakeLists.txt") + version = "3.22.1" + } } buildTypes { diff --git a/plugins/Python-Tools/python-bundle.lock b/plugins/Python-Tools/python-bundle.lock new file mode 100644 index 00000000..05ff7673 --- /dev/null +++ b/plugins/Python-Tools/python-bundle.lock @@ -0,0 +1,31 @@ +wheelhouse black-26.5.1-py3-none-any.whl https://files.pythonhosted.org/packages/94/51/f975cae76d44274cc2868dc9040ac5d58d464784610234455b4e7b19c6ef/black-26.5.1-py3-none-any.whl 4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2 +wheelhouse blinker-1.9.0-py3-none-any.whl https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc +wheelhouse click-8.5.0-py3-none-any.whl https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl 255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 +wheelhouse docstring_to_markdown-0.17-py3-none-any.whl https://files.pythonhosted.org/packages/56/7b/af3d0da15bed3a8665419bb3a630585756920f4ad67abfdfef26240ebcc0/docstring_to_markdown-0.17-py3-none-any.whl fd7d5094aa83943bf5f9e1a13701866b7c452eac19765380dead666e36d3711c +wheelhouse flask-3.1.3-py3-none-any.whl https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c +wheelhouse gunicorn-26.2.0-py3-none-any.whl https://files.pythonhosted.org/packages/fe/85/7522a52e5e2f42faf1a129113ab63e548c42e103e9af395b7bfe65e403e2/gunicorn-26.2.0-py3-none-any.whl bd249d0b3f7972f7432f0a6b6ff3b3ee2d129f70cd1ff6c09a9dd9e29a2b88e3 +wheelhouse importlib_metadata-9.0.1-py3-none-any.whl https://files.pythonhosted.org/packages/b3/55/ecca97ae19075f1fac62def77731e7f535e6c1fb8f92ff08160c5e6dade8/importlib_metadata-9.0.1-py3-none-any.whl bba5600596a7e21f3eef53281cf28d6a5195634d2f2b78ff9501a3272c6eaab0 +wheelhouse itsdangerous-2.2.0-py3-none-any.whl https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef +wheelhouse jedi-0.20.0-py2.py3-none-any.whl https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl 7bdd9c2634f56713299976f4cbd59cb3fa92165cc5e05ea811fb253480728b67 +wheelhouse jinja2-3.1.6-py3-none-any.whl https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl 85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +wheelhouse markupsafe-3.0.3.tar.gz https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz 722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 +wheelhouse mypy_extensions-1.1.0-py3-none-any.whl https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl 1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 +wheelhouse packaging-26.3-py3-none-any.whl https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +wheelhouse parso-0.8.7-py2.py3-none-any.whl https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c +wheelhouse pathspec-1.1.1-py3-none-any.whl https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 +wheelhouse platformdirs-4.12.2-py3-none-any.whl https://files.pythonhosted.org/packages/d0/89/446044f33aba0348d35e433f56d12206d010a5281a1df54054d4cfb82388/platformdirs-4.12.2-py3-none-any.whl 29dbf06d96c500bc6bdbce75fb0a14d63279c93b1842f97e72a135b33e856983 +wheelhouse pluggy-1.6.0-py3-none-any.whl https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 +wheelhouse pyflakes-3.2.0-py2.py3-none-any.whl https://files.pythonhosted.org/packages/d4/d7/f1b7db88d8e4417c5d47adad627a93547f44bdc9028372dbd2313f34a855/pyflakes-3.2.0-py2.py3-none-any.whl 84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a +wheelhouse python_lsp_jsonrpc-1.1.2-py3-none-any.whl https://files.pythonhosted.org/packages/cb/d9/656659d5b5d5f402b2b174cd0ba9bc827e07ce3c0bf88da65424baf64af8/python_lsp_jsonrpc-1.1.2-py3-none-any.whl 7339c2e9630ae98903fdaea1ace8c47fba0484983794d6aafd0bd8989be2b03c +wheelhouse python_lsp_server-1.15.0-py3-none-any.whl https://files.pythonhosted.org/packages/c2/4a/b4abc04dcc65c985fa66d6486393ad3051ff485ccabd9727aea43a242f88/python_lsp_server-1.15.0-py3-none-any.whl d6ac11b467021310498f2dffb2edf09629bbcfe0d3073156db2337334a60463f +wheelhouse pytokens-0.4.1-py3-none-any.whl https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl 26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de +wheelhouse setuptools-84.0.0-py3-none-any.whl https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl 51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 +wheelhouse setuptools_scm-10.3.4-py3-none-any.whl https://files.pythonhosted.org/packages/da/f5/54538a1f17ea753c42b0928dc2e62986d463f8c393340b9a1e602f2d4dd9/setuptools_scm-10.3.4-py3-none-any.whl 82f34c3e3084fc2b57d397200637cc13f2338004052d2fee47baa5f0e902464e +wheelhouse typing_extensions-4.16.0-py3-none-any.whl https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl 481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 +wheelhouse ujson-6.0.0.tar.gz https://files.pythonhosted.org/packages/64/7c/e1fa3fb70b53192436d751b5cb671f0ee960baa188b8351a7fec735223d3/ujson-6.0.0.tar.gz 80e23393feb707582e0ad495c397a4477b646d08094d2df64f7316f9fafd8aae +wheelhouse vcs_versioning-2.5.0-py3-none-any.whl https://files.pythonhosted.org/packages/e4/e6/b4dedd1efea8a1e432123575896328dec83c26327164f7098db415d13f6f/vcs_versioning-2.5.0-py3-none-any.whl dbf44f6303dc817e5792cb7d0e9cc90e3046ee1e32eaa357b20ad368811313d0 +wheelhouse werkzeug-3.1.9-py3-none-any.whl https://files.pythonhosted.org/packages/a1/38/df03f564f43cec2684823f3cccae1a652ee7face1cbaa76fb223096e64d7/werkzeug-3.1.9-py3-none-any.whl 6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab +wheelhouse whatthepatch-1.0.7-py3-none-any.whl https://files.pythonhosted.org/packages/8e/93/af1d6ccb69ab6b5a00e03fa0cefa563f9862412667776ea15dd4eece3a90/whatthepatch-1.0.7-py3-none-any.whl 1b6f655fd31091c001c209529dfaabbabdbad438f5de14e3951266ea0fc6e7ed +wheelhouse wheel-0.48.0-py3-none-any.whl https://files.pythonhosted.org/packages/2e/29/69cfbb602cd91690c55d38ba9fe53e6a7e76a6fa647bf38f19c138d25449/wheel-0.48.0-py3-none-any.whl 3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab +wheelhouse yapf-0.43.0-py3-none-any.whl https://files.pythonhosted.org/packages/37/81/6acd6601f61e31cfb8729d3da6d5df966f80f374b78eff83760714487338/yapf-0.43.0-py3-none-any.whl 224faffbc39c428cb095818cf6ef5511fdab6f7430a10783fdfb292ccf2852ca +wheelhouse zipp-4.1.0-py3-none-any.whl https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl 25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f diff --git a/plugins/Python-Tools/src/main/AndroidManifest.xml b/plugins/Python-Tools/src/main/AndroidManifest.xml index a6cbf79c..fbfb343f 100644 --- a/plugins/Python-Tools/src/main/AndroidManifest.xml +++ b/plugins/Python-Tools/src/main/AndroidManifest.xml @@ -27,11 +27,11 @@ + android:value="26.41" /> + android:value="filesystem.read,filesystem.write,system.commands,project.structure,native.code" /> " + "+=" + "<" + "<<" + "<<=" + "<=" + "<>" + "=" + ":=" + "==" + ">" + ">=" + ">>" + ">>=" + "|" + "|=" + "~" + "@=" + "and" + "in" + "is" + "not" + "or" + "is not" + "not in" +] @operator + +[ + "as" + "assert" + "async" + "await" + "break" + "class" + "continue" + "def" + "del" + "elif" + "else" + "except" + "exec" + "finally" + "for" + "from" + "global" + "if" + "import" + "lambda" + "nonlocal" + "pass" + "print" + "raise" + "return" + "try" + "while" + "with" + "yield" + "match" + "case" +] @keyword + +; Identifier naming conventions + +((identifier) @constant + (#match? @constant "^[A-Z][A-Z_]*$")) + +((identifier) @constructor + (#match? @constructor "^[A-Z]")) + +(identifier) @variable diff --git a/plugins/Python-Tools/src/main/cpp/CMakeLists.txt b/plugins/Python-Tools/src/main/cpp/CMakeLists.txt new file mode 100644 index 00000000..8c93e2e8 --- /dev/null +++ b/plugins/Python-Tools/src/main/cpp/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.22.1) +project(tree_sitter_grammar C) + +set(GRAMMAR_TARGET tree-sitter-${TREE_SITTER_GRAMMAR_NAME}) +set(GRAMMAR_SOURCE_DIR ${TREE_SITTER_GRAMMAR_DIR}/src) +file(GLOB GRAMMAR_SOURCES ${GRAMMAR_SOURCE_DIR}/parser.c ${GRAMMAR_SOURCE_DIR}/scanner.c) + +add_library(${GRAMMAR_TARGET} SHARED ${GRAMMAR_SOURCES}) +target_include_directories(${GRAMMAR_TARGET} PRIVATE ${GRAMMAR_SOURCE_DIR}) +set_target_properties(${GRAMMAR_TARGET} PROPERTIES C_STANDARD 11) +target_link_options(${GRAMMAR_TARGET} PRIVATE -Wl,-z,max-page-size=16384) diff --git a/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt b/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt index 3b401d48..c40de6b3 100644 --- a/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt +++ b/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt @@ -12,9 +12,13 @@ import com.itsaky.androidide.plugins.extensions.BuildActionExtension import com.itsaky.androidide.plugins.extensions.CommandResult import com.itsaky.androidide.plugins.extensions.CommandSpec import com.itsaky.androidide.plugins.extensions.DocumentationExtension +import com.itsaky.androidide.plugins.extensions.LanguageDefinition +import com.itsaky.androidide.plugins.extensions.LanguageExtension +import com.itsaky.androidide.plugins.extensions.LanguageServerDefinition import com.itsaky.androidide.plugins.extensions.PluginBuildAction import com.itsaky.androidide.plugins.extensions.PluginTooltipEntry import com.itsaky.androidide.plugins.extensions.ToolbarActionIds +import com.itsaky.androidide.plugins.extensions.TreeSitterGrammar import com.itsaky.androidide.plugins.services.IdeCommandService import com.itsaky.androidide.plugins.services.IdeEditorService import com.itsaky.androidide.plugins.services.IdeProjectService @@ -30,6 +34,8 @@ import kotlinx.coroutines.launch import java.io.File import java.lang.ref.WeakReference import java.util.concurrent.atomic.AtomicBoolean +import java.util.zip.ZipInputStream +import java.util.concurrent.atomic.AtomicReference /** * Python Tools plugin. @@ -44,7 +50,7 @@ import java.util.concurrent.atomic.AtomicBoolean * plugin must not steal the toolbar from Java/Kotlin/Android projects, and it must be usable on a * fresh device without Python pre-installed. */ -class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension { +class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension, LanguageExtension { private var pluginContext: PluginContext? = null private var templateService: IdeTemplateService? = null @@ -58,6 +64,7 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension private var installJob: Job? = null private var toolbarContainer: WeakReference? = null private val tooltipBindingScheduled = AtomicBoolean(false) + private val requirementsSyncedRoot = AtomicReference(null) override fun initialize(context: PluginContext): Boolean { pluginContext = context @@ -74,7 +81,7 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension override fun activate(): Boolean { installPyHooks() registerTemplates() - installJob = scope.launch { ensurePython() } + installJob = scope.launch { if (ensurePython()) ensureBundledPackages() } Log.i(TAG, "Python Tools activated") return true } @@ -89,6 +96,18 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension return true } + override fun getLanguages(): List = listOf( + LanguageDefinition( + languageId = LANGUAGE_ID, + fileExtensions = setOf("py"), + grammar = TreeSitterGrammar(name = LANGUAGE_ID, queriesAssetPath = "treesitter/python"), + server = LanguageServerDefinition( + command = listOf(LANGUAGE_SERVER_COMMAND), + environment = mapOf("PYTHONUNBUFFERED" to "1"), + ), + ), + ) + override fun dispose() { scope.cancel() pluginContext = null @@ -104,6 +123,7 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension override fun toolbarActionsToHide(): Set { if (!isPythonProjectOpen()) return emptySet() + syncRequirementsOnce() scheduleTooltipBinding() return ToolbarActionIds.BUILD_HIDEABLE } @@ -154,7 +174,7 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension description = "Install dependencies from requirements.txt", icon = R.drawable.ic_sync_deps, category = BuildActionCategory.BUILD, - command = shell("pip install -r requirements.txt"), + command = shell(requirementsInstallScript()), timeoutMs = 300_000, ), ) @@ -217,12 +237,7 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension } notify("Missing dependencies — installing from requirements.txt…") scope.launch { - val result = try { - cmd.executeCommand(shell("pip install -r requirements.txt"), timeoutMs = 300_000L).await() - } catch (t: Throwable) { - notify("Could not start dependency install.") - return@launch - } + val result = pipInstallRequirements(cmd) ?: return@launch when (result) { is CommandResult.Success -> notify("Dependencies installed. Tap Run again.") is CommandResult.Failure -> notify("Dependency install failed (exit ${result.exitCode}).") @@ -231,6 +246,39 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension } } + private fun syncRequirementsOnce() { + val cmd = commandService ?: return + val root = projectService?.getCurrentProject()?.rootDir ?: return + val path = root.absolutePath + if (requirementsSyncedRoot.getAndSet(path) == path) return + scope.launch { + installJob?.join() + if (!File(root, "requirements.txt").exists()) return@launch + val result = pipInstallRequirements(cmd) ?: return@launch + when (result) { + is CommandResult.Success -> + if (result.stdout.contains("Successfully installed")) notify("Project dependencies installed.") + is CommandResult.Failure -> { + requirementsSyncedRoot.compareAndSet(path, null) + Log.e(TAG, "pip install -r requirements.txt failed: ${result.stderr.take(2000)}") + notify("Could not install project dependencies (exit ${result.exitCode}). It needs a network connection.") + } + is CommandResult.Cancelled -> requirementsSyncedRoot.compareAndSet(path, null) + } + } + } + + private suspend fun pipInstallRequirements(cmd: IdeCommandService): CommandResult? = try { + cmd.executeCommand( + shell(requirementsInstallScript()), + timeoutMs = 300_000L, + ).await() + } catch (t: Throwable) { + Log.e(TAG, "Could not start pip", t) + notify("Could not start dependency install.") + null + } + private fun isPythonProjectOpen(): Boolean = PythonDomain.isPythonProject(projectService?.getCurrentProject()?.rootDir) @@ -355,39 +403,111 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension // region Python interpreter bootstrap - private suspend fun ensurePython() { + private suspend fun ensurePython(): Boolean { val cmd = commandService ?: run { Log.e(TAG, "IdeCommandService unavailable; cannot manage Python") - return + return false } if (pythonAvailable(cmd)) { Log.i(TAG, "Python is already installed") - return + return true } notify("Python not found. Installing via Termux…") - val result = try { - cmd.executeCommand(shell("pkg install python -y"), timeoutMs = 15 * 60_000L).await() - } catch (t: Throwable) { - Log.e(TAG, "Could not start the package manager", t) - notify("Could not start the package manager (pkg). Install Python manually.") + return runInstall(cmd, "Python", "pkg install python -y", INSTALL_TIMEOUT_MS) && pythonAvailable(cmd) + } + + private suspend fun ensureBundledPackages() { + val cmd = commandService ?: return + if (!extractWheelhouse()) { + notify("Could not unpack the bundled Python packages.") + return + } + if (probe(cmd, "command -v $LANGUAGE_SERVER_COMMAND >/dev/null && python -c 'import flask, gunicorn'")) { + Log.i(TAG, "Bundled Python packages are already installed") return } - when (result) { - is CommandResult.Success -> - if (pythonAvailable(cmd)) notify("Python installed successfully.") - else notify("Install finished but Python is still not on PATH.") + notify("Installing the Python language server, Flask, and gunicorn. This runs once.") + val wheelhouse = quote(wheelhouseDir().absolutePath) + runInstall( + cmd, + "the Python language server, Flask, and gunicorn", + "python -m pip install --disable-pip-version-check --no-index --find-links $wheelhouse " + + "\"$LANGUAGE_SERVER_PACKAGE\" $BUNDLED_PACKAGES", + INSTALL_TIMEOUT_MS, + ) + } + + private suspend fun runInstall( + cmd: IdeCommandService, + label: String, + script: String, + timeoutMs: Long, + ): Boolean { + val result = try { + cmd.executeCommand(shell(script), timeoutMs = timeoutMs).await() + } catch (t: Throwable) { + Log.e(TAG, "Could not start the $label install", t) + notify("Could not start the $label install.") + return false + } + return when (result) { + is CommandResult.Success -> { + notify("Installed $label.") + true + } is CommandResult.Failure -> { val detail = result.error ?: result.stderr.takeIf { it.isNotBlank() } ?: "exit ${result.exitCode}" - Log.e(TAG, "pkg install python failed: $detail") - notify("Failed to install Python: $detail") + Log.e(TAG, "$label install failed: $detail") + notify("Failed to install $label: ${detail.lineSequence().last { it.isNotBlank() }}") + false + } + is CommandResult.Cancelled -> { + Log.i(TAG, "$label install cancelled") + false + } + } + } + + private suspend fun probe(cmd: IdeCommandService, script: String): Boolean = try { + val result = cmd.executeCommand(shell(script), timeoutMs = PROBE_TIMEOUT_MS).await() + result is CommandResult.Success && result.exitCode == 0 + } catch (t: Throwable) { + false + } + + private fun extractWheelhouse(): Boolean { + val assets = pluginContext?.androidContext?.assets ?: return false + val target = wheelhouseDir() + return runCatching { + target.mkdirs() + ZipInputStream(assets.open(WHEELHOUSE_ASSET)).use { zip -> + generateSequence { zip.nextEntry }.forEach { entry -> + val out = File(target, entry.name) + if (!out.canonicalPath.startsWith(target.canonicalPath + File.separator)) { + throw SecurityException("Wheelhouse entry escapes its directory: ${entry.name}") + } + if (!out.exists()) out.outputStream().use { zip.copyTo(it) } + } } - is CommandResult.Cancelled -> Log.i(TAG, "Python installation cancelled") + true + }.getOrElse { + Log.e(TAG, "Failed to extract the bundled wheelhouse", it) + false } } + private fun pluginDir(): File = pluginContext!!.resources.getPluginDirectory() + + private fun wheelhouseDir(): File = File(pluginDir(), "wheelhouse") + + private fun quote(path: String): String = "\"" + path + "\"" + + private fun requirementsInstallScript(): String = + "python -m pip install --disable-pip-version-check --find-links ${quote(wheelhouseDir().absolutePath)} -r requirements.txt" + private suspend fun pythonAvailable(cmd: IdeCommandService): Boolean = try { val result = cmd.executeCommand(shell("python --version"), timeoutMs = 20_000L).await() result is CommandResult.Success && result.exitCode == 0 @@ -407,6 +527,13 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension companion object { private const val TAG = "PythonToolsPlugin" + private const val LANGUAGE_ID = "python" + private const val LANGUAGE_SERVER_COMMAND = "pylsp" + private const val LANGUAGE_SERVER_PACKAGE = "python-lsp-server[pyflakes,yapf]" + private const val PROBE_TIMEOUT_MS = 20_000L + private const val INSTALL_TIMEOUT_MS = 20 * 60_000L + private const val WHEELHOUSE_ASSET = "python/wheelhouse.zip" + private const val BUNDLED_PACKAGES = "flask gunicorn" internal const val PLUGIN_ID = "com.appdevforall.python.plugin" internal const val ACTION_RUN_APP = "python.run.app" From 30267425351d244340f739e7c53504e46ba2cfa5 Mon Sep 17 00:00:00 2001 From: Daniel Alome Date: Mon, 5 Oct 2026 16:36:02 +0100 Subject: [PATCH 2/3] ADFA-4851 | Python-Tools: address review of the language support - A failed or cancelled requirements install no longer re-arms the automatic sync. toolbarActionsToHide() runs on every toolbar refresh, so an offline project re-ran a 300 s pip install and re-showed the failure toast on nearly every refresh. The sync now runs once per project open; the failure toast points at Install requirements for a retry. - downloadTreeSitterGrammar declares the grammar version, URL and sha256 as task inputs, so changing any of them re-runs the download instead of the task staying up to date on a stale grammar. - The wheelhouse is extracted through .partial and renamed into place, so a copy interrupted by a kill or a full disk never leaves a truncated wheel at the final name for later activations to skip and pip to reject. --- plugins/Python-Tools/build.gradle.kts | 3 +++ .../python/plugin/PythonToolsPlugin.kt | 15 +++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/plugins/Python-Tools/build.gradle.kts b/plugins/Python-Tools/build.gradle.kts index d758c140..41234fb4 100644 --- a/plugins/Python-Tools/build.gradle.kts +++ b/plugins/Python-Tools/build.gradle.kts @@ -138,6 +138,9 @@ val downloadGrammar = val source = grammarSource val outputDir = grammarDir val archive = layout.buildDirectory.file("tree-sitter/${source.name}-${source.version}.tgz") + inputs.property("grammarVersion", source.version) + inputs.property("grammarUrl", source.url) + inputs.property("grammarSha256", source.sha256) outputs.dir(outputDir) doLast { diff --git a/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt b/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt index c40de6b3..3c573961 100644 --- a/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt +++ b/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt @@ -32,6 +32,7 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.launch import java.io.File +import java.io.IOException import java.lang.ref.WeakReference import java.util.concurrent.atomic.AtomicBoolean import java.util.zip.ZipInputStream @@ -259,11 +260,13 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension, is CommandResult.Success -> if (result.stdout.contains("Successfully installed")) notify("Project dependencies installed.") is CommandResult.Failure -> { - requirementsSyncedRoot.compareAndSet(path, null) Log.e(TAG, "pip install -r requirements.txt failed: ${result.stderr.take(2000)}") - notify("Could not install project dependencies (exit ${result.exitCode}). It needs a network connection.") + notify( + "Could not install project dependencies (exit ${result.exitCode}). It needs a network " + + "connection; tap ${actionLabel(ACTION_SYNC_DEPS)} to try again.", + ) } - is CommandResult.Cancelled -> requirementsSyncedRoot.compareAndSet(path, null) + is CommandResult.Cancelled -> Log.i(TAG, "pip install -r requirements.txt cancelled") } } } @@ -489,7 +492,11 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension, if (!out.canonicalPath.startsWith(target.canonicalPath + File.separator)) { throw SecurityException("Wheelhouse entry escapes its directory: ${entry.name}") } - if (!out.exists()) out.outputStream().use { zip.copyTo(it) } + if (!out.exists()) { + val partial = File(target, "${entry.name}.partial") + partial.outputStream().use { zip.copyTo(it) } + if (!partial.renameTo(out)) throw IOException("Could not move ${partial.name} into place") + } } } true From fc011537a1412e95596e1aaeb9ce5bfc31ce360a Mon Sep 17 00:00:00 2001 From: Daniel Alome Date: Mon, 5 Oct 2026 21:45:13 +0100 Subject: [PATCH 3/3] ADFA-4851 | Python-Tools: hide Quick Build in Python projects toolbarActionsToHide() can only hide the Gradle actions in ToolbarActionIds.BUILD_HIDEABLE, which do not include Quick Build (ide.editor.build.quickBuild), so it stayed on the toolbar of a Python project. Implement UIExtension and return it from getHiddenToolbarActionIds() while a Python project is open, as Go-Tools does. --- plugins/Python-Tools/python-tools.html | 6 ++++++ .../com/appdevforall/python/plugin/PythonToolsPlugin.kt | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/plugins/Python-Tools/python-tools.html b/plugins/Python-Tools/python-tools.html index 8f2e3691..79288eec 100644 --- a/plugins/Python-Tools/python-tools.html +++ b/plugins/Python-Tools/python-tools.html @@ -231,6 +231,12 @@

Hiding the Gradle actions

override fun toolbarActionsToHide(): Set<String> =
     if (isPythonProjectOpen()) ToolbarActionIds.BUILD_HIDEABLE else emptySet()
+

+ Quick Run is not in that set. The plugin hides it through + UIExtension, which accepts any toolbar action id: +

+ 
override fun getHiddenToolbarActionIds(): Set<String> =
+    if (isPythonProjectOpen()) setOf("ide.editor.build.quickBuild") else emptySet()

Process lifecycle

diff --git a/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt b/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt index 3c573961..e734d626 100644 --- a/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt +++ b/plugins/Python-Tools/src/main/kotlin/com/appdevforall/python/plugin/PythonToolsPlugin.kt @@ -19,6 +19,7 @@ import com.itsaky.androidide.plugins.extensions.PluginBuildAction import com.itsaky.androidide.plugins.extensions.PluginTooltipEntry import com.itsaky.androidide.plugins.extensions.ToolbarActionIds import com.itsaky.androidide.plugins.extensions.TreeSitterGrammar +import com.itsaky.androidide.plugins.extensions.UIExtension import com.itsaky.androidide.plugins.services.IdeCommandService import com.itsaky.androidide.plugins.services.IdeEditorService import com.itsaky.androidide.plugins.services.IdeProjectService @@ -51,7 +52,7 @@ import java.util.concurrent.atomic.AtomicReference * plugin must not steal the toolbar from Java/Kotlin/Android projects, and it must be usable on a * fresh device without Python pre-installed. */ -class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension, LanguageExtension { +class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension, LanguageExtension, UIExtension { private var pluginContext: PluginContext? = null private var templateService: IdeTemplateService? = null @@ -129,6 +130,9 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension, return ToolbarActionIds.BUILD_HIDEABLE } + override fun getHiddenToolbarActionIds(): Set = + if (isPythonProjectOpen()) setOf(QUICK_BUILD_ACTION_ID) else emptySet() + override fun getBuildActions(): List { if (!isPythonProjectOpen()) return emptyList() scheduleTooltipBinding() @@ -541,6 +545,7 @@ class PythonToolsPlugin : IPlugin, BuildActionExtension, DocumentationExtension, private const val INSTALL_TIMEOUT_MS = 20 * 60_000L private const val WHEELHOUSE_ASSET = "python/wheelhouse.zip" private const val BUNDLED_PACKAGES = "flask gunicorn" + private const val QUICK_BUILD_ACTION_ID = "ide.editor.build.quickBuild" internal const val PLUGIN_ID = "com.appdevforall.python.plugin" internal const val ACTION_RUN_APP = "python.run.app"