diff --git a/.gitignore b/.gitignore index 227f773..7b228e0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ gdpr_consent/node_modules/ *~ .streamlit/secrets.toml docs/superpowers/ +.venv/ +pr-397.patch \ No newline at end of file diff --git a/.streamlit/config.toml b/.streamlit/config.toml index f54abde..9b41e52 100644 --- a/.streamlit/config.toml +++ b/.streamlit/config.toml @@ -10,7 +10,7 @@ developmentMode = false files = ["/app/admin-secrets/secrets.toml", "~/.streamlit/secrets.toml", ".streamlit/secrets.toml"] [server] -maxUploadSize = 1000 #MB +maxUploadSize = 3000 #MB port = 8501 # should be same as configured in deployment repo diff --git a/CLAUDE.md b/CLAUDE.md index 948210c..e0e0a22 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,94 +1,173 @@ -# CLAUDE.md +# OpenMS Streamlit WebApp Template -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +## What This Is -## Project Overview +**This is the standard framework for building web applications for mass spectrometry (MS) data analysis**, used across the OpenMS ecosystem for proteomics and metabolomics research. When a researcher or developer needs a web-based tool for MS data processing, visualization, or analysis — whether for label-free quantification, untargeted metabolomics, top-down proteomics, or any other MS workflow — this template is how it gets built. -OpenMS Streamlit Template is a web application framework for building mass spectrometry (MS) analysis workflows using OpenMS/pyOpenMS. It supports both simple pyOpenMS workflows and complex multi-tool pipelines using OpenMS TOPP (The OpenMS Proteomics Pipeline) tools. +The template wraps **OpenMS/pyOpenMS** (the leading open-source C++/Python library for computational mass spectrometry) and its **TOPP tools** (a suite of ~200 command-line tools for MS data processing pipelines) into interactive Streamlit web applications. -## Common Commands +### Production Apps Built From This Template -```bash -# Run the app locally -streamlit run app.py +- **OpenMS/quantms-web** — quantitative proteomics (DDA-LFQ, DDA-ISO, DIA-LFQ quantification) +- **OpenMS/umetaflow** — untargeted metabolomics (feature detection, alignment, annotation, GNPS molecular networking) +- **OpenMS/FLASHApp** — top-down proteomics (FLASHDeconv deconvolution result visualization) -# Run tests -python -m pytest test_gui.py tests/ +### Mass Spectrometry Domain Context -# Build and run with Docker (includes OpenMS TOPP tools) -docker-compose up -d --build +- **Input data** is typically mzML (raw MS spectra), featureXML (detected features), consensusXML (linked features across samples), idXML (peptide/protein identifications), traML (targeted transitions) +- **Typical workflows chain TOPP tools**: e.g., `FeatureFinderMetabo` (detect LC-MS features) → `FeatureLinkerUnlabeledKD` (align features across runs) → custom Python post-processing +- **Proteomics** focuses on peptide/protein identification and quantification (tools like `MSGFPlusAdapter`, `FidoAdapter`, `ProteinQuantifier`) +- **Metabolomics** focuses on feature detection, annotation, and statistical analysis (tools like `FeatureFinderMetabo`, `MetaboliteAdductDecharger`, `SiriusAdapter`) +- **pyOpenMS** provides Python bindings for programmatic MS data access — reading mzML files, manipulating spectra/chromatograms, computing molecular properties, etc. +- **MS-specific visualizations**: mass spectra (m/z vs intensity), chromatograms (RT vs intensity), peak maps (RT vs m/z 2D heatmaps), isotope patterns, fragment ion annotations, volcano plots for differential expression + +## Architecture -# Clean up old workspaces (removes workspaces older than 7 days) -python clean-up-workspaces.py +``` +app.py # Entry point — registers pages via st.Page() in a dict +settings.json # App config: name, version, deployment mode, threading +default-parameters.json # Default workspace parameters (tracked via widget keys) +presets.json # Parameter presets for TOPP workflows +content/ # Streamlit pages (one .py per page) +src/ + common/common.py # Utilities: page_setup(), save_params(), show_fig(), show_table() + Workflow.py # Example WorkflowManager subclass (TOPP workflow) + workflow/ + WorkflowManager.py # Base class: upload/configure/execution/results pattern + StreamlitUI.py # Widget library: upload_widget, input_TOPP, input_python, etc. + ParameterManager.py # JSON parameter persistence + TOPP .ini generation + CommandExecutor.py # Runs TOPP tools and Python scripts as subprocesses + FileManager.py # Workspace file organization + Logger.py # Structured workflow logging + QueueManager.py # Redis queue for online deployments + python-tools/ # Custom Python analysis scripts (with DEFAULTS dicts) +Dockerfile # Full build: OpenMS + TOPP tools + pyOpenMS +Dockerfile_simple # Lightweight: pyOpenMS only +docker-compose.yml # Deployment config ``` -Note: Local runs have limited functionality. Features requiring OpenMS TOPP tools only work out of the box with Docker or when OpenMS Command Line Tools are installed separately. +## Key Patterns -## Architecture +### Pages -### Core Framework (`src/workflow/`) +Every page starts with `page_setup()` which handles workspace initialization, sidebar rendering, and parameter loading: -The workflow system is built around `WorkflowManager` as the base class with these components: +```python +from src.common.common import page_setup, save_params +params = page_setup() +``` -- **WorkflowManager**: Base class that orchestrates file management, parameters, command execution, and UI. Custom workflows inherit from this and override `upload()`, `configure()`, `execution()`, and `results()` methods. -- **FileManager**: Handles input/output file organization in `workflow_dir/input-files/{key}/` and `workflow_dir/results/` -- **ParameterManager**: Manages TOPP tool parameters (XML .ini files) and JSON parameters -- **CommandExecutor**: Runs external commands (TOPP tools) with threading for parallelization -- **StreamlitUI**: Provides Streamlit widgets including `upload_widget()` and `input_TOPP()` for TOPP parameter UIs -- **Logger**: Multi-level logging (minimal, commands, all) to `workflow_dir/logs/` +Pages are registered in `app.py` under named sections: -### Page Structure +```python +pages = { + "Section Name": [ + st.Page(Path("content", "my_page.py"), title="My Page", icon="🔬"), + ], +} +``` -- **Entry point**: `app.py` defines multi-page navigation using `st.navigation()` -- **Pages**: Each file in `content/` is a page that calls `page_setup()` from `src/common/common.py`, then instantiates a workflow class -- **Utility pages** (`content/`): digest.py, fragmentation.py, isotope_pattern_generator.py, peptide_mz_calculator.py provide standalone analysis tools +### Parameters -### Workflow Data Flow +Parameters are tracked via widget keys that match entries in `default-parameters.json`. The `save_params(params)` call at the end of a page persists any widget state changes: + +```python +params = page_setup() +st.number_input("X", value=params["my-param"], key="my-param") +save_params(params) +``` -1. `page_setup()` initializes workspace and loads parameters from `workspace/params.json` -2. Workflow class (e.g., `WorkflowTest`) inherits from `WorkflowManager` -3. Each page calls the appropriate method: `show_file_upload_section()`, `show_parameter_section()`, `show_execution_section()`, `show_results_section()` -4. Workflow execution runs in a multiprocessing.Process to avoid blocking Streamlit UI updates +### TOPP Workflows (WorkflowManager) -### Key Patterns +Complex workflows subclass `WorkflowManager` and implement 4 methods: +- `upload()` — file upload widgets via `self.ui.upload_widget()` +- `configure()` — TOPP params via `self.ui.input_TOPP()`, Python tool params via `self.ui.input_python()` +- `execution()` — run tools via `self.executor.run_topp()` and `self.executor.run_python()` +- `results()` — display outputs -- **Workspace isolation**: Each user session gets a unique workspace directory for files and parameters -- **Streamlit fragments**: Use `@st.fragment` decorator for interactive UI updates without full page reloads -- **TOPP tool execution**: `executor.run_topp("ToolName", {inputs/outputs}, {extra_params})` handles parameter files and command construction +Each workflow gets 4 content pages (upload, configure, run, results) that call `wf.show_*_section()`. -## Configuration Files +Decorate `configure()` and `results()` with `@st.fragment` for partial reruns. -- `settings.json`: App name, version, analytics, workspace settings -- `default-parameters.json`: Workflow default parameters -- `.streamlit/config.toml`: Streamlit server config (port 8501, 1000MB upload limit) +For conditional UI (a widget that shows/hides other widgets), pass `reactive=True` to `input_widget`, `select_input_file`, or `input_TOPP` so a change reruns the parent `configure()` instead of only its isolated fragment. Read the changed value from `st.session_state` (not `self.params`, which is stale within the rerun) via `parameter_manager.param_prefix` for custom-widget keys or `topp_param_prefix` for TOPP keys (`":1:"`). -## Creating New Workflows +### Python Tools -Inherit from `WorkflowManager` and implement the four core methods: +Custom scripts in `src/python-tools/` define a `DEFAULTS` list for auto-generated UI: ```python -from src.workflow.WorkflowManager import WorkflowManager +DEFAULTS = [ + {"key": "in", "value": [], "hide": True}, + {"key": "my-param", "value": 5, "name": "My Parameter", "help": "Description", + "min": 1, "max": 100, "step_size": 1, "widget_type": "slider"}, +] +``` -class MyWorkflow(WorkflowManager): - def __init__(self): - super().__init__("My Workflow", st.session_state["workspace"]) +### Presets - def upload(self): - self.ui.upload_widget(key="input-files", name="Input", file_types="mzML", fallback=[...]) +Parameter presets in `presets.json` map workflow names (lowercase, hyphens) to named parameter sets: + +```json +{ + "workflow-name": { + "Preset Name": { + "_description": "Tooltip text", + "TOPPToolName": {"algorithm:section:param": value}, + "_general": {"custom-key": value} + } + } +} +``` - def configure(self): - self.ui.input_TOPP("ToolName", custom_defaults={...}, include_parameters=[...]) +## Visualization Libraries - def execution(self): - self.executor.run_topp("ToolName", {"in": [...], "out": [...]}, {...}) +Two libraries are commonly used in template-based apps for MS data visualization: + +### pyopenms-viz + +Pandas DataFrame extension for MS visualization. Use the plotly backend in Streamlit: + +```python +import pyopenms_viz +df.plot.ms_spectrum(backend="plotly") # mass spectrum (m/z vs intensity) +df.plot.peak_map(backend="plotly") # 2D peak map (RT vs m/z heatmap) +df.plot.chromatogram(backend="plotly") # chromatogram (RT vs intensity) +df.plot.mobilogram(backend="plotly") # ion mobility trace +``` + +Best for: publication-quality static/interactive plots, small-medium datasets, standard MS plot types. + +### OpenMS-Insight (t0mdavid-m/openms-insight) + +Vue.js-backed interactive Streamlit components for large MS datasets: + +- `Table` — server-side pagination with Tabulator.js +- `LinePlot` — stick-style mass spectra via Plotly +- `Heatmap` — 2D scatter handling millions of points +- `VolcanoPlot` — differential expression visualization +- `SequenceView` — peptide sequence with fragment ion matching + +Components support cross-linking via shared identifiers. Best for: large datasets (millions of points), cross-component interactivity, server-side pagination. + +## Commands + +```bash +# Run locally +pip install -r requirements.txt +streamlit run app.py + +# Run tests +python -m pytest tests/ - def results(self): - st.dataframe(pd.read_csv(...)) +# Docker +docker-compose up --build ``` -## Key Dependencies +## Conventions -- **pyOpenMS 3.5.0+**: Python bindings for OpenMS -- **Streamlit 1.43.0**: Web UI framework -- **Plotly + streamlit_plotly_events**: Interactive visualizations -- **OpenMS TOPP tools**: External command-line tools (Docker or separate install required) +- Page files go in `content/`, source logic in `src/` +- Widget keys must match parameter keys in `default-parameters.json` +- Workflow names use lowercase with hyphens: "My Workflow" -> "my-workflow" +- Use `show_fig()` and `show_table()` from `src/common/common.py` for consistent display +- Use `@st.fragment` on methods that should partially rerun (configure, results) +- TOPP tool parameters use colon-separated paths: `"algorithm:section:param_name"` diff --git a/Dockerfile b/Dockerfile index 2d72377..2d1b5da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ ARG PORT=8501 # Streamlit app GitHub user name (to download artifact from). ARG GITHUB_USER=OpenMS # Streamlit app GitHub repository name (to download artifact from). -ARG GITHUB_REPO=quantms-web +ARG GITHUB_REPO=streamlit-template USER root @@ -47,6 +47,13 @@ RUN wget -q \ && rm -f Miniforge3-Linux-x86_64.sh RUN mamba --version +# Make /root traversable so the entrypoint can `source +# /root/miniforge3/bin/activate ...` when the container runs as a non-root +# user (apptainer/singularity maps the host UID into the container; the +# default ubuntu /root is 0700 which would block path traversal). +x only, +# not +r, so the directory listing remains private. +RUN chmod o+x /root + # Setup mamba environment. RUN mamba create -n streamlit-env python=3.10 RUN echo "mamba activate streamlit-env" >> ~/.bashrc @@ -78,14 +85,20 @@ RUN mkdir /openms-build WORKDIR /openms-build # Configure. -RUN /bin/bash -c "cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF -DPYOPENMS=OFF ../OpenMS" +RUN /bin/bash -c "cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF -DPYOPENMS=ON ../OpenMS -DPY_MEMLEAK_DISABLE=On" # Build TOPP tools and clean up. RUN make -j4 TOPP RUN rm -rf src doc CMakeFiles -# Install dependencies (pyopenms will be installed from pip) -COPY requirements.txt ./requirements.txt +# Build pyOpenMS wheels and install via pip. +RUN make -j4 pyopenms +WORKDIR /openms-build/pyOpenMS +RUN pip install dist/*.whl + +# Install other dependencies (excluding pyopenms) +COPY requirements.txt ./requirements.txt +RUN grep -Ev '^pyopenms([=<>!~].*)?$' requirements.txt > requirements_cleaned.txt && mv requirements_cleaned.txt requirements.txt RUN pip install -r requirements.txt WORKDIR / @@ -110,12 +123,26 @@ RUN rm -rf openms-build # Prepare and run streamlit app. FROM compile-openms AS run-app -# Install Redis server for job queue and nginx for load balancing +# Install Redis server for job queue and nginx for load balancing. +# Redis data lives under $RUNTIME_DIR at runtime (see entrypoint.sh) so no +# /var/lib/redis setup is needed - that path is not writable under Apptainer. RUN apt-get update && apt-get install -y --no-install-recommends redis-server nginx \ && rm -rf /var/lib/apt/lists/* -# Create Redis data directory -RUN mkdir -p /var/lib/redis && chown redis:redis /var/lib/redis +# Create Redis data directory. Default 0755 root-owned is enough: the docker +# entrypoint runs as root (can write regardless of mode), and the apptainer +# entrypoint relocates Redis state to /tmp/openms-runtime-* so this dir is +# never written under apptainer. +RUN mkdir -p /var/lib/redis + +# Pre-create bind-mount targets so apptainer/singularity has a real attach +# point. Docker auto-creates missing `-v` targets, but singularity uses a +# read-only underlay and silently ignores `:rw` when the target isn't a +# real directory in the SIF — writes then fail with EROFS even though the +# host bind path is writable. Pre-creating these directories costs one +# inode each and changes nothing in docker mode (the user's volume mount +# shadows them). +RUN mkdir -p /workspaces-streamlit-template /mounted-data # Create workdir and copy over all streamlit related files/folders. @@ -123,6 +150,7 @@ RUN mkdir -p /var/lib/redis && chown redis:redis /var/lib/redis WORKDIR /app COPY assets/ /app/assets COPY content/ /app/content +COPY docs/ /app/docs COPY example-data/ /app/example-data COPY gdpr_consent/ /app/gdpr_consent COPY hooks/ /app/hooks @@ -148,67 +176,10 @@ ENV REDIS_URL=redis://localhost:6379/0 # Set to >1 to enable nginx load balancer with multiple Streamlit instances ENV STREAMLIT_SERVER_COUNT=1 -# create entrypoint script to start cron, Redis, RQ workers, and Streamlit -RUN echo -e '#!/bin/bash\n\ -set -e\n\ -source /root/miniforge3/bin/activate streamlit-env\n\ -\n\ -# Start cron for workspace cleanup\n\ -service cron start\n\ -\n\ -# Start Redis server in background\n\ -echo "Starting Redis server..."\n\ -redis-server --daemonize yes --dir /var/lib/redis --appendonly no\n\ -\n\ -# Wait for Redis to be ready\n\ -until redis-cli ping > /dev/null 2>&1; do\n\ - echo "Waiting for Redis..."\n\ - sleep 1\n\ -done\n\ -echo "Redis is ready"\n\ -\n\ -# Start RQ worker(s) in background\n\ -WORKER_COUNT=${RQ_WORKER_COUNT:-1}\n\ -echo "Starting $WORKER_COUNT RQ worker(s)..."\n\ -for i in $(seq 1 $WORKER_COUNT); do\n\ - rq worker openms-workflows --url $REDIS_URL --name worker-$i &\n\ -done\n\ -\n\ -# Load balancer setup\n\ -SERVER_COUNT=${STREAMLIT_SERVER_COUNT:-1}\n\ -\n\ -if [ "$SERVER_COUNT" -gt 1 ]; then\n\ - echo "Starting $SERVER_COUNT Streamlit instances with nginx load balancer..."\n\ -\n\ - # Generate nginx upstream block\n\ - UPSTREAM_SERVERS=""\n\ - BASE_PORT=8510\n\ - for i in $(seq 0 $((SERVER_COUNT - 1))); do\n\ - PORT=$((BASE_PORT + i))\n\ - UPSTREAM_SERVERS="${UPSTREAM_SERVERS} server 127.0.0.1:${PORT};\\n"\n\ - done\n\ -\n\ - # Write nginx config\n\ - mkdir -p /etc/nginx\n\ - echo -e "worker_processes auto;\\npid /run/nginx.pid;\\n\\nevents {\\n worker_connections 1024;\\n}\\n\\nhttp {\\n client_max_body_size 0;\\n\\n map \\$cookie_stroute \\$route_key {\\n \\x22\\x22 \\$request_id;\\n default \\$cookie_stroute;\\n }\\n\\n upstream streamlit_backend {\\n hash \\$route_key consistent;\\n${UPSTREAM_SERVERS} }\\n\\n map \\$http_upgrade \\$connection_upgrade {\\n default upgrade;\\n \\x27\\x27 close;\\n }\\n\\n server {\\n listen 0.0.0.0:8501;\\n\\n location / {\\n proxy_pass http://streamlit_backend;\\n proxy_http_version 1.1;\\n proxy_set_header Upgrade \\$http_upgrade;\\n proxy_set_header Connection \\$connection_upgrade;\\n proxy_set_header Host \\$host;\\n proxy_set_header X-Real-IP \\$remote_addr;\\n proxy_set_header X-Forwarded-For \\$proxy_add_x_forwarded_for;\\n proxy_set_header X-Forwarded-Proto \\$scheme;\\n proxy_read_timeout 86400;\\n proxy_send_timeout 86400;\\n proxy_buffering off;\\n add_header Set-Cookie \\x22stroute=\\$route_key; Path=/; HttpOnly; SameSite=Lax\\x22 always;\\n }\\n }\\n}" > /etc/nginx/nginx.conf\n\ -\n\ - # Start Streamlit instances on internal ports\n\ - for i in $(seq 0 $((SERVER_COUNT - 1))); do\n\ - PORT=$((BASE_PORT + i))\n\ - echo "Starting Streamlit instance on port $PORT..."\n\ - streamlit run app.py --server.port $PORT --server.address 0.0.0.0 &\n\ - done\n\ -\n\ - sleep 2\n\ - echo "Starting nginx load balancer on port 8501..."\n\ - exec /usr/sbin/nginx -g "daemon off;"\n\ -else\n\ - # Single instance mode (default) - run Streamlit directly on port 8501\n\ - echo "Starting Streamlit app..."\n\ - exec streamlit run app.py --server.address 0.0.0.0\n\ -fi\n\ -' > /app/entrypoint.sh -# make the script executable +# Install the apptainer-compatible entrypoint that starts cron (when the root +# FS is writable), Redis, RQ workers, optional nginx load balancer, and the +# Streamlit server. The script falls back to /tmp paths under apptainer. +COPY docker/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh # Patch Analytics @@ -217,6 +188,11 @@ RUN mamba run -n streamlit-env python hooks/hook-analytics.py # Set Online Deployment RUN jq '.online_deployment = true' settings.json > tmp.json && mv tmp.json settings.json +# Point the in-app mounted-drive browser at the conventional bind-mount path. +# The browser only renders when this directory exists at runtime, i.e. when +# the user starts the container with `-v /host/path:/mounted-data`. +RUN jq '.local_data_dir = "/mounted-data"' settings.json > tmp.json && mv tmp.json settings.json + # Download latest OpenMS App executable as a ZIP file. # ARG declared here (not at the top) — otherwise the per-run token busts the cache. ARG GITHUB_TOKEN diff --git a/Dockerfile.arm b/Dockerfile.arm new file mode 100644 index 0000000..1765980 --- /dev/null +++ b/Dockerfile.arm @@ -0,0 +1,237 @@ +# This Dockerfile builds OpenMS, the TOPP tools, pyOpenMS and thidparty tools. +# It also adds a basic streamlit server that serves a pyOpenMS-based app. +# hints: +# build image and give it a name (here: streamlitapp) with: docker build -f Dockerfile.arm --no-cache -t streamlitapp:latest-arm64 --build-arg GITHUB_TOKEN= . 2>&1 | tee build.log +# check if image was build: docker image ls +# run container: docker run -p 8501:8501 streamlitappsimple:latest +# debug container after build (comment out ENTRYPOINT) and run container with interactive /bin/bash shell +# prune unused images/etc. to free disc space (e.g. might be needed on gitpod). Use with care.: docker system prune --all --force + +FROM ubuntu:22.04 AS setup-build-system +ARG OPENMS_REPO=https://github.com/OpenMS/OpenMS.git +ARG OPENMS_BRANCH=release/3.5.0 +ARG PORT=8501 +# Streamlit app GitHub user name (to download artifact from). +ARG GITHUB_USER=OpenMS +# Streamlit app GitHub repository name (to download artifact from). +ARG GITHUB_REPO=streamlit-template + +USER root + +# Install required Ubuntu packages. +RUN apt-get -y update +RUN apt-get install -y --no-install-recommends --no-install-suggests g++ autoconf automake patch libtool make git gpg wget ca-certificates curl jq libgtk2.0-dev openjdk-8-jdk cron cmake +RUN update-ca-certificates +RUN apt-get install -y --no-install-recommends --no-install-suggests libsvm-dev libeigen3-dev coinor-libcbc-dev libglpk-dev libzip-dev zlib1g-dev libxerces-c-dev libbz2-dev libomp-dev libhdf5-dev +RUN apt-get install -y --no-install-recommends --no-install-suggests libboost-date-time1.74-dev \ + libboost-iostreams1.74-dev \ + libboost-regex1.74-dev \ + libboost-math1.74-dev \ + libboost-random1.74-dev +RUN apt-get install -y --no-install-recommends --no-install-suggests qt6-base-dev libqt6svg6-dev libqt6opengl6-dev libqt6openglwidgets6 libgl-dev + +# Install Github CLI +RUN (type -p wget >/dev/null || (apt-get update && apt-get install wget -y)) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt-get update \ + && apt-get install gh -y + +# Download and install miniforge. +ENV PATH="/root/miniforge3/bin:${PATH}" +RUN wget -q \ + https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh \ + && bash Miniforge3-Linux-aarch64.sh -b \ + && rm -f Miniforge3-Linux-aarch64.sh +RUN mamba --version + +# Make /root traversable so the entrypoint can `source +# /root/miniforge3/bin/activate ...` when the container runs as a non-root +# user (apptainer/singularity maps the host UID into the container; the +# default ubuntu /root is 0700 which would block path traversal). +x only, +# not +r, so the directory listing remains private. +RUN chmod o+x /root + +# Setup mamba environment. +RUN mamba create -n streamlit-env python=3.10 +RUN echo "mamba activate streamlit-env" >> ~/.bashrc +SHELL ["/bin/bash", "--rcfile", "~/.bashrc"] +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +# Install up-to-date cmake via mamba and packages for pyOpenMS build. +RUN mamba install cmake +RUN pip install --upgrade pip && python -m pip install -U setuptools nose cython "autowrap<=0.24" pandas numpy pytest + +# Clone OpenMS branch and the associcated contrib+thirdparties+pyOpenMS-doc submodules. +RUN git clone --recursive --depth=1 -b ${OPENMS_BRANCH} --single-branch ${OPENMS_REPO} && cd /OpenMS + +# Pull Linux compatible third-party dependencies and store them in directory thirdparty. +WORKDIR /OpenMS +RUN mkdir /thirdparty && \ + git submodule update --init THIRDPARTY && \ + cp -r THIRDPARTY/All/* /thirdparty && \ + if [ -d "THIRDPARTY/Linux/aarch64" ]; then \ + cp -r THIRDPARTY/Linux/aarch64/* /thirdparty; \ + fi && \ + chmod -R +x /thirdparty +ENV PATH="/thirdparty/LuciPHOr2:/thirdparty/MSGFPlus:/thirdparty/ThermoRawFileParser:/thirdparty/Comet:/thirdparty/Percolator:/thirdparty/Sage:${PATH}" + +# Build OpenMS and pyOpenMS. +FROM setup-build-system AS compile-openms +WORKDIR / + +# Set up build directory. +RUN mkdir /openms-build +WORKDIR /openms-build + +# Configure (two-pass — mirrors FLASHApp.arm). +# Pass 1 runs under plain bash so cmake does NOT search /root/miniforge3 +# when resolving C++ system dependencies. On ARM the conda-forge build of +# libyaml-cpp.so.0.8 is linked against a newer libstdc++ (GLIBCXX_3.4.32, +# i.e. gcc 13+) than ubuntu:22.04's system g++ ships, so letting cmake +# pick the miniforge yaml-cpp makes TOPP linking fail with +# undefined reference to `std::ios_base_library_init()@GLIBCXX_3.4.32` +# amd64 happens to work because its conda-forge yaml-cpp build is older. +# We call the mamba-env cmake by full path so we get a version >= 3.24 +# (OpenMS 3.5's floor); ubuntu:22.04's apt cmake is 3.22 which is too old. +# CMAKE_IGNORE_PREFIX_PATH keeps cmake from auto-discovering miniforge libs +# even though the binary itself lives there. +# Pass 2 re-runs cmake inside the mamba env with PYOPENMS=ON so the Python +# bindings can find the conda-forge Python/Cython/NumPy; CMAKE_IGNORE_PREFIX_PATH +# keeps the C++ link command unchanged from pass 1. +SHELL ["/bin/bash", "-c"] +RUN /root/miniforge3/envs/streamlit-env/bin/cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DCMAKE_IGNORE_PREFIX_PATH=/root/miniforge3 -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF ../OpenMS +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] +RUN cmake -DPYOPENMS=ON -DPY_MEMLEAK_DISABLE=On -DCMAKE_IGNORE_PREFIX_PATH=/root/miniforge3 . + +# Build TOPP tools and clean up. +RUN make -j4 TOPP +# NOTE: do NOT delete CMakeFiles/ here. The two-pass cmake configure used +# above generates CMakeFiles/VerifyGlobs.cmake for the pyOpenMS targets' +# CONFIGURE_DEPENDS globs; the next `make -j4 pyopenms` runs +# `cmake_check_build_system` which fails fast if VerifyGlobs.cmake is gone: +# CMake Error: Not a file: /openms-build/CMakeFiles/VerifyGlobs.cmake +# The x86 single-pass build seems to avoid generating that file (different +# cmake codepath when PYOPENMS is set during the initial configure), which +# is why it can still `rm -rf CMakeFiles` here. CMakeFiles/ adds ~a few +# hundred MB to the intermediate layer — acceptable. +RUN rm -rf src doc + +# Build pyOpenMS wheels and install via pip. +RUN make -j4 pyopenms +WORKDIR /openms-build/pyOpenMS +RUN pip install dist/*.whl + +# Install other dependencies (excluding pyopenms) +COPY requirements.txt ./requirements.txt +RUN grep -Ev '^pyopenms([=<>!~].*)?$' requirements.txt > requirements_cleaned.txt && mv requirements_cleaned.txt requirements.txt +RUN pip install -r requirements.txt + +WORKDIR / +RUN mkdir openms + +# Copy TOPP tools bin directory, add to PATH. +RUN cp -r openms-build/bin /openms/bin +ENV PATH="/openms/bin/:${PATH}" + +# Copy TOPP tools bin directory, add to PATH. +RUN cp -r openms-build/lib /openms/lib +ENV LD_LIBRARY_PATH="/openms/lib/:${LD_LIBRARY_PATH}" + +# Copy share folder, add to PATH, remove source directory. +RUN cp -r OpenMS/share/OpenMS /openms/share +RUN rm -rf OpenMS +ENV OPENMS_DATA_PATH="/openms/share/" + +# Remove build directory. +RUN rm -rf openms-build + +# Prepare and run streamlit app. +FROM compile-openms AS run-app + +# Install Redis server for job queue and nginx for load balancing. +# Redis data lives under $RUNTIME_DIR at runtime (see entrypoint.sh) so no +# /var/lib/redis setup is needed - that path is not writable under Apptainer. +RUN apt-get update && apt-get install -y --no-install-recommends redis-server nginx \ + && rm -rf /var/lib/apt/lists/* + +# Create Redis data directory. Default 0755 root-owned is enough: the docker +# entrypoint runs as root (can write regardless of mode), and the apptainer +# entrypoint relocates Redis state to /tmp/openms-runtime-* so this dir is +# never written under apptainer. +RUN mkdir -p /var/lib/redis + +# Pre-create bind-mount targets so apptainer/singularity has a real attach +# point. Docker auto-creates missing `-v` targets, but singularity uses a +# read-only underlay and silently ignores `:rw` when the target isn't a +# real directory in the SIF — writes then fail with EROFS even though the +# host bind path is writable. Pre-creating these directories costs one +# inode each and changes nothing in docker mode (the user's volume mount +# shadows them). +RUN mkdir -p /workspaces-streamlit-template /mounted-data + +# Create workdir and copy over all streamlit related files/folders. + +# note: specifying folder with slash as suffix and repeating the folder name seems important to preserve directory structure +WORKDIR /app +COPY assets/ /app/assets +COPY content/ /app/content +COPY docs/ /app/docs +COPY example-data/ /app/example-data +COPY gdpr_consent/ /app/gdpr_consent +COPY hooks/ /app/hooks +COPY src/ /app/src +COPY utils/ /app/utils +COPY app.py /app/app.py +COPY settings.json /app/settings.json +COPY default-parameters.json /app/default-parameters.json +COPY presets.json /app/presets.json + +# For streamlit configuration +COPY .streamlit/ /app/.streamlit/ +COPY clean-up-workspaces.py /app/clean-up-workspaces.py + +# add cron job to the crontab +RUN echo "0 3 * * * /root/miniforge3/envs/streamlit-env/bin/python /app/clean-up-workspaces.py >> /app/clean-up-workspaces.log 2>&1" | crontab - + +# Set default worker count (can be overridden via environment variable) +ENV RQ_WORKER_COUNT=1 +ENV REDIS_URL=redis://localhost:6379/0 + +# Number of Streamlit server instances for load balancing (default: 1 = no load balancer) +# Set to >1 to enable nginx load balancer with multiple Streamlit instances +ENV STREAMLIT_SERVER_COUNT=1 + +# Install the apptainer-compatible entrypoint that starts cron (when the root +# FS is writable), Redis, RQ workers, optional nginx load balancer, and the +# Streamlit server. The script falls back to /tmp paths under apptainer. +COPY docker/entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh + +# Patch Analytics +RUN mamba run -n streamlit-env python hooks/hook-analytics.py + +# Set Online Deployment +RUN jq '.online_deployment = true' settings.json > tmp.json && mv tmp.json settings.json + +# Point the in-app mounted-drive browser at the conventional bind-mount path. +# The browser only renders when this directory exists at runtime, i.e. when +# the user starts the container with `-v /host/path:/mounted-data`. +RUN jq '.local_data_dir = "/mounted-data"' settings.json > tmp.json && mv tmp.json settings.json + +# Download latest OpenMS App executable as a ZIP file. +# ARG declared here (not at the top) — otherwise the per-run token busts the cache. +ARG GITHUB_TOKEN +RUN if [ -n "$GITHUB_TOKEN" ]; then \ + echo "GITHUB_TOKEN is set, proceeding to download the release asset..."; \ + gh release download -R ${GITHUB_USER}/${GITHUB_REPO} -p "OpenMS-App.zip" -D /app; \ + else \ + echo "GITHUB_TOKEN is not set, skipping the release asset download."; \ + fi + + +# Run app as container entrypoint. +EXPOSE $PORT +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/Dockerfile_simple b/Dockerfile_simple new file mode 100644 index 0000000..163bcfe --- /dev/null +++ b/Dockerfile_simple @@ -0,0 +1,127 @@ +# This Dockerfile creates a container with pyOpenMS +# It also adds a basic streamlit server that serves a pyOpenMS-based app. +# hints: +# build image with: docker build -f Dockerfile_simple --no-cache -t streamlitapp:latest --build-arg GITHUB_TOKEN= . 2>&1 | tee build.log +# check if image was build: docker image ls +# run container: docker run -p 8501:8501 streamlitapp:latest +# debug container after build (comment out ENTRYPOINT) and run container with interactive /bin/bash shell +# prune unused images/etc. to free disc space (e.g. might be needed on gitpod). Use with care.: docker system prune --all --force + +FROM ubuntu:22.04 AS stage1 +ARG OPENMS_REPO=https://github.com/OpenMS/OpenMS.git +ARG OPENMS_BRANCH=develop +ARG PORT=8501 +# Streamlit app GitHub user name (to download artifact from). +ARG GITHUB_USER=OpenMS +# Streamlit app GitHub repository name (to download artifact from). +ARG GITHUB_REPO=streamlit-template + + +# Step 1: set up a sane build system +USER root + +RUN apt-get -y update +# note: streamlit in docker needs libgtk2.0-dev (see https://yugdamor.medium.com/importerror-libgthread-2-0-so-0-cannot-open-shared-object-file-no-such-file-or-directory-895b94a7827b) +RUN apt-get install -y --no-install-recommends --no-install-suggests wget ca-certificates libgtk2.0-dev curl jq cron nginx +RUN update-ca-certificates + +# Install Github CLI +RUN (type -p wget >/dev/null || (apt-get update && apt-get install wget -y)) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt-get update \ + && apt-get install gh -y + +# Download and install miniforge. +ENV PATH="/root/miniforge3/bin:${PATH}" +RUN wget -q \ + https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh \ + && bash Miniforge3-Linux-x86_64.sh -b \ + && rm -f Miniforge3-Linux-x86_64.sh +RUN mamba --version + +# Make /root traversable so the entrypoint can `source +# /root/miniforge3/bin/activate ...` when the container runs as a non-root +# user (apptainer/singularity maps the host UID into the container; the +# default ubuntu /root is 0700 which would block path traversal). +x only, +# not +r, so the directory listing remains private. +RUN chmod o+x /root + +# Setup mamba environment. +RUN mamba create -n streamlit-env python=3.10 +RUN echo "mamba activate streamlit-env" >> ~/.bashrc +SHELL ["/bin/bash", "--rcfile", "~/.bashrc"] +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +#################################### install streamlit +# install packages +COPY requirements.txt requirements.txt +RUN mamba install pip +RUN python -m pip install --upgrade pip +RUN python -m pip install -r requirements.txt + +# Pre-create bind-mount targets so apptainer/singularity has a real attach +# point. Docker auto-creates missing `-v` targets, but singularity uses a +# read-only underlay and silently ignores `:rw` when the target isn't a +# real directory in the SIF — writes then fail with EROFS even though the +# host bind path is writable. +RUN mkdir -p /workspaces-streamlit-template /mounted-data + +# create workdir and copy over all streamlit related files/folders +WORKDIR /app +# note: specifying folder with slash as suffix and repeating the folder name seems important to preserve directory structure +WORKDIR /app +COPY assets/ /app/assets +COPY content/ /app/content +COPY docs/ /app/docs +COPY example-data/ /app/example-data +COPY gdpr_consent/ /app/gdpr_consent +COPY hooks/ /app/hooks +COPY src/ /app/src +COPY utils/ /app/utils +COPY app.py /app/app.py +COPY settings.json /app/settings.json +COPY default-parameters.json /app/default-parameters.json +COPY presets.json /app/presets.json + +# For streamlit configuration +COPY .streamlit/ /app/.streamlit/ + +COPY clean-up-workspaces.py /app/clean-up-workspaces.py + +# add cron job to the crontab +RUN echo "0 3 * * * /root/miniforge3/envs/streamlit-env/bin/python /app/clean-up-workspaces.py >> /app/clean-up-workspaces.log 2>&1" | crontab - + +# Number of Streamlit server instances for load balancing (default: 1 = no load balancer) +# Set to >1 to enable nginx load balancer with multiple Streamlit instances +ENV STREAMLIT_SERVER_COUNT=1 + +# Install the apptainer-compatible entrypoint (shared with the full image). +# The script auto-skips the Redis/RQ section when redis-server is not +# installed, so it works equally well in the simple variant. +COPY docker/entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh + +# Patch Analytics +RUN mamba run -n streamlit-env python hooks/hook-analytics.py + +# Set Online Deployment +RUN jq '.online_deployment = true' settings.json > tmp.json && mv tmp.json settings.json + +# Download latest OpenMS App executable as a ZIP file. +# ARG declared here (not at the top) — otherwise the per-run token busts the cache. +ARG GITHUB_TOKEN +RUN if [ -n "$GITHUB_TOKEN" ]; then \ + echo "GITHUB_TOKEN is set, proceeding to download the release asset..."; \ + gh release download -R ${GITHUB_USER}/${GITHUB_REPO} -p "OpenMS-App.zip" -D /app; \ + else \ + echo "GITHUB_TOKEN is not set, skipping the release asset download."; \ + fi + +# make sure that mamba environment is used +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +EXPOSE $PORT +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/Dockerfile_simple.arm b/Dockerfile_simple.arm new file mode 100644 index 0000000..be57317 --- /dev/null +++ b/Dockerfile_simple.arm @@ -0,0 +1,127 @@ +# This Dockerfile creates a container with pyOpenMS +# It also adds a basic streamlit server that serves a pyOpenMS-based app. +# hints: +# build image with: docker build -f Dockerfile_simple.arm --no-cache -t streamlitapp:latest-arm64 --build-arg GITHUB_TOKEN= . 2>&1 | tee build.log +# check if image was build: docker image ls +# run container: docker run -p 8501:8501 streamlitapp:latest +# debug container after build (comment out ENTRYPOINT) and run container with interactive /bin/bash shell +# prune unused images/etc. to free disc space (e.g. might be needed on gitpod). Use with care.: docker system prune --all --force + +FROM ubuntu:22.04 AS stage1 +ARG OPENMS_REPO=https://github.com/OpenMS/OpenMS.git +ARG OPENMS_BRANCH=develop +ARG PORT=8501 +# Streamlit app GitHub user name (to download artifact from). +ARG GITHUB_USER=OpenMS +# Streamlit app GitHub repository name (to download artifact from). +ARG GITHUB_REPO=streamlit-template + + +# Step 1: set up a sane build system +USER root + +RUN apt-get -y update +# note: streamlit in docker needs libgtk2.0-dev (see https://yugdamor.medium.com/importerror-libgthread-2-0-so-0-cannot-open-shared-object-file-no-such-file-or-directory-895b94a7827b) +RUN apt-get install -y --no-install-recommends --no-install-suggests wget ca-certificates libgtk2.0-dev curl jq cron nginx +RUN update-ca-certificates + +# Install Github CLI +RUN (type -p wget >/dev/null || (apt-get update && apt-get install wget -y)) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt-get update \ + && apt-get install gh -y + +# Download and install miniforge. +ENV PATH="/root/miniforge3/bin:${PATH}" +RUN wget -q \ + https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh \ + && bash Miniforge3-Linux-aarch64.sh -b \ + && rm -f Miniforge3-Linux-aarch64.sh +RUN mamba --version + +# Make /root traversable so the entrypoint can `source +# /root/miniforge3/bin/activate ...` when the container runs as a non-root +# user (apptainer/singularity maps the host UID into the container; the +# default ubuntu /root is 0700 which would block path traversal). +x only, +# not +r, so the directory listing remains private. +RUN chmod o+x /root + +# Setup mamba environment. +RUN mamba create -n streamlit-env python=3.10 +RUN echo "mamba activate streamlit-env" >> ~/.bashrc +SHELL ["/bin/bash", "--rcfile", "~/.bashrc"] +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +#################################### install streamlit +# install packages +COPY requirements.txt requirements.txt +RUN mamba install pip +RUN python -m pip install --upgrade pip +RUN python -m pip install -r requirements.txt + +# Pre-create bind-mount targets so apptainer/singularity has a real attach +# point. Docker auto-creates missing `-v` targets, but singularity uses a +# read-only underlay and silently ignores `:rw` when the target isn't a +# real directory in the SIF — writes then fail with EROFS even though the +# host bind path is writable. +RUN mkdir -p /workspaces-streamlit-template /mounted-data + +# create workdir and copy over all streamlit related files/folders +WORKDIR /app +# note: specifying folder with slash as suffix and repeating the folder name seems important to preserve directory structure +WORKDIR /app +COPY assets/ /app/assets +COPY content/ /app/content +COPY docs/ /app/docs +COPY example-data/ /app/example-data +COPY gdpr_consent/ /app/gdpr_consent +COPY hooks/ /app/hooks +COPY src/ /app/src +COPY utils/ /app/utils +COPY app.py /app/app.py +COPY settings.json /app/settings.json +COPY default-parameters.json /app/default-parameters.json +COPY presets.json /app/presets.json + +# For streamlit configuration +COPY .streamlit/ /app/.streamlit/ + +COPY clean-up-workspaces.py /app/clean-up-workspaces.py + +# add cron job to the crontab +RUN echo "0 3 * * * /root/miniforge3/envs/streamlit-env/bin/python /app/clean-up-workspaces.py >> /app/clean-up-workspaces.log 2>&1" | crontab - + +# Number of Streamlit server instances for load balancing (default: 1 = no load balancer) +# Set to >1 to enable nginx load balancer with multiple Streamlit instances +ENV STREAMLIT_SERVER_COUNT=1 + +# Install the apptainer-compatible entrypoint (shared with the full image). +# The script auto-skips the Redis/RQ section when redis-server is not +# installed, so it works equally well in the simple variant. +COPY docker/entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh + +# Patch Analytics +RUN mamba run -n streamlit-env python hooks/hook-analytics.py + +# Set Online Deployment +RUN jq '.online_deployment = true' settings.json > tmp.json && mv tmp.json settings.json + +# Download latest OpenMS App executable as a ZIP file. +# ARG declared here (not at the top) — otherwise the per-run token busts the cache. +ARG GITHUB_TOKEN +RUN if [ -n "$GITHUB_TOKEN" ]; then \ + echo "GITHUB_TOKEN is set, proceeding to download the release asset..."; \ + gh release download -R ${GITHUB_USER}/${GITHUB_REPO} -p "OpenMS-App.zip" -D /app; \ + else \ + echo "GITHUB_TOKEN is not set, skipping the release asset download."; \ + fi + +# make sure that mamba environment is used +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +EXPOSE $PORT +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/app.py b/app.py index 97f9a16..e76dcd8 100644 --- a/app.py +++ b/app.py @@ -1,3 +1,10 @@ +import os +# Polars' default (CPU-core-count-sized) native thread pool access-violation +# crashes the whole process on some high-core-count Windows machines when +# invoked from Streamlit's script-runner thread. Must be set before polars +# is imported anywhere (including transitively via openms_insight). +os.environ.setdefault("POLARS_MAX_THREADS", "1") + import streamlit as st from pathlib import Path import json @@ -23,12 +30,19 @@ st.Page(Path("content", "results_rescoring.py"), title="Rescoring", icon="📈"), st.Page(Path("content", "results_filtered.py"), title="Filtered PSMs", icon="🎯"), st.Page(Path("content", "results_abundance.py"), title="Abundance", icon="📋"), + + ], + "Differential Protein Analysis": [ + st.Page(Path("content", "filtering.py"), title="Filtering", icon="🧹"), + st.Page(Path("content", "imputation.py"), title="Imputation", icon="🩹"), + st.Page(Path("content", "normalization.py"), title="Normalization", icon="⚖️"), + st.Page(Path("content", "statistical.py"), title="Statistical", icon="🔢"), st.Page(Path("content", "results_volcano.py"), title="Volcano", icon="🌋"), st.Page(Path("content", "results_pca.py"), title="PCA", icon="📊"), st.Page(Path("content", "results_heatmap.py"), title="Heatmap", icon="🔥"), - st.Page(Path("content", "results_library.py"), title="Spectral Library", icon="📚"), - st.Page(Path("content", "results_pathway_analysis.py"), title="Pathway Analysis", icon="📉"), - ], + st.Page(Path("content", "results_heatmap_clustered.py"), title="Clustered Heatmap", icon="🧬"), + st.Page(Path("content", "enrichment.py"), title="Pathway Analysis", icon="📉"), + ] } pg = st.navigation(pages) diff --git a/content/enrichment.py b/content/enrichment.py new file mode 100644 index 0000000..62fc9fa --- /dev/null +++ b/content/enrichment.py @@ -0,0 +1,141 @@ +"""Pathway Analysis Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column +# Import GO Enrichment modules from openms_insight engine +from openms_insight.analysis.enrichment import calculate_go_enrichment + +params = page_setup() +st.title("GO Enrichment Analysis") + +st.markdown( + """ +Identify overrepresented biological themes (BP, CC, MF) within your differentially expressed protein features using MyGene.info and Fisher's Exact Test. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# --- STEP 1: Upstream Statistics Checkpoint --- +if ( + "statistics_df" in st.session_state + and st.session_state["statistics_df"] is not None +): + final_statistics_report = st.session_state["statistics_df"] + st.info( + "🔄 **Upstream Pipeline Detected**: Using analyzed matrices from the **Statistical Inference** step." + ) +else: + st.warning( + "⚠️ **Missing Prerequisites**: Statistical inference data not detected. Please run hypothesis testing first." + ) + st.page_link( + "content/statistical.py", label="Go to Statistical Inference", icon="🔬" + ) + st.stop() + +# --- STEP 2: Preprocessing Mapping Key Configuration --- +# Identify target identifier columns dynamically +abundance_result = get_abundance_data(st.session_state["workspace"]) +id_col = get_id_column(st.session_state["workspace"], abundance_result[0]) if abundance_result else "ProteinName" +if id_col not in final_statistics_report.columns: + st.error(f"❌ Structural Error: Column '{id_col}' is missing from the active matrix context.") + st.stop() + +# --- SECTION 1: Parameter Setup & Dynamic Cutoff Labels --- +st.subheader("Configure Enrichment Thresholds") + +# Check if target p-value should be adjusted or raw based on previous selections (Fallback safely to 'p-adj') +target_p_col = "p-adj" if "p-adj" in final_statistics_report.columns else "p-value" +p_label = ( + "Adjusted P-value (p-adj) Cutoff" + if target_p_col == "p-adj" + else "Raw P-value (p-value) Cutoff" +) + +ui_go_col1, ui_go_col2 = st.columns(2) + +with ui_go_col1: + p_cutoff = st.number_input( + f"🔬 {p_label}", + min_value=0.0001, + max_value=1.0, + value=0.05, + step=0.01, + format="%.4f", + help="Proteins with significance metrics below this value are mapped to the foreground cohort.", + ) + +with ui_go_col2: + fc_cutoff = st.number_input( + "📈 Absolute Difference Cutoff (|log2FC|)", + min_value=0.0, + max_value=10.0, + value=1.0, + step=0.1, + format="%.2f", + help="Proteins with absolute log2 fold change greater than or equal to this threshold will be selected.", + ) + +# --- SECTION 2: Execution and Interactive View Charts --- +st.markdown("
", unsafe_allow_html=True) +if st.button("🚀 Run GO Enrichment Analysis", type="primary", key="run_go_analysis"): + + with st.spinner("Querying MyGene.info API & executing hyper-geometric calculation loops..."): + # Convert internal pandas DataFrame to openms_insight Polars DataFrame expectation + stats_pl = pl.from_pandas(final_statistics_report) + + status, output = calculate_go_enrichment( + final_report=stats_pl, + id_col=id_col, + target_p_col=target_p_col, + p_cutoff=p_cutoff, + fc_cutoff=fc_cutoff, + ) + + # Route response structures based on analysis output status code + if status == "empty_data": + st.error("❌ No valid statistical rows found containing standard columns to run GO alignment.") + + elif status == "insufficient_proteins": + st.warning( + f"⚠️ Not enough significant proteins found to construct target datasets. " + f"(Criteria: {target_p_col} < {p_cutoff:.4f}, |log2FC| ≥ {fc_cutoff:.2f})." + ) + st.info(f"💡 Found significant proteins count: **{output}**. Try relaxing your p-value or log2FC filters.") + + elif status == "success": + st.success("⭕ GO Enrichment Analysis completed successfully!") + + # Display operational matrix scale + st.markdown( + f"📊 **Analysis Profile Scope**: Mapped **{output['fg_count']}** significant foreground profiles out of **{output['bg_count']}** reference background items." + ) + + # Build multi-tab interface layer for ontology subcategories + tabs = st.tabs([ + "🧬 Biological Process (BP)", + "🔬 Cellular Component (CC)", + "🧪 Molecular Function (MF)" + ]) + categories_data = output["categories"] + + for idx, go_type in enumerate(["BP", "CC", "MF"]): + with tabs[idx]: + fig = categories_data[go_type]["fig"] + df_go = categories_data[go_type]["df"] + + if fig is not None and df_go is not None: + # Render plotly bar figures generated straight from backend engine + st.plotly_chart(fig, use_container_width=True) + + st.subheader(f"📊 {go_type} Results Dataframe") + st.dataframe(df_go, use_container_width=True) + else: + st.info(f"No statistically overrepresented terms identified for Category: **{go_type}**") \ No newline at end of file diff --git a/content/filtering.py b/content/filtering.py new file mode 100644 index 0000000..2bad00c --- /dev/null +++ b/content/filtering.py @@ -0,0 +1,173 @@ +"""Filtering Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import filtering functions from openms_insight package +from openms_insight.analysis.filter import ( + filter_low_abundance, + filter_low_repeatability, + filter_low_variance, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Filtering") + +st.markdown( + """ +Filter out low-quality proteins from your dataset based on abundance, repeatability, or variance thresholds. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="📋" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Identify actual sample columns dynamically +sample_cols = [ + c + for c in pivot_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Original Data View --- +st.subheader("Original Abundance Table") +st.markdown( + f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering." +) +st.dataframe(pivot_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Filter Configuration --- +st.subheader("Configure Filter Engine") + +# Prepare Polars Metadata DataFrame required by openms_insight functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for filtering strategy +filter_method = st.selectbox( + "Select Filtering Method", + options=["Low Abundance", "Low Repeatability", "Low Variance"], + index=0, + help="Choose the statistical criteria to prune unreliable protein entries.", +) + +# Render threshold sliders dynamically based on the selected filter method +if filter_method == "Low Abundance": + st.markdown( + "**Low Abundance Filter**: Keeps rows where at least one group's median is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +elif filter_method == "Low Repeatability": + st.markdown( + "**Low Repeatability Filter**: Keeps rows where at least one group has a missing value ratio within the allowed maximum." + ) + threshold = st.slider( + "Max Missing Ratio", + min_value=0.0, + max_value=100.0, + value=50.0, + step=5.0, + help="Allowed missing value (zero or null) ratio per group.", + ) + +elif filter_method == "Low Variance": + st.markdown( + "**Low Variance Filter**: Keeps rows where at least one group's variance is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +# --- SECTION 3: Filter Execution and Collected Results View --- +if st.button("Apply Filter", type="primary"): + # Convert the original Pandas DataFrame into a Polars LazyFrame graph + quant_lazy = pl.from_pandas(pivot_df).lazy() + + # Route execution to the chosen openms_insight engine function + if filter_method == "Low Abundance": + filtered_lazy = filter_low_abundance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + elif filter_method == "Low Repeatability": + # Convert percent slider input to ratio expected by the function (e.g., 50.0% -> 0.5) + filtered_lazy = filter_low_repeatability( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + max_missing_ratio=threshold / 100.0, + ) + elif filter_method == "Low Variance": + filtered_lazy = filter_low_variance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + + # Collect the evaluated lazy graph and convert back to Pandas for visualization + filtered_df = strip_stat_columns(filtered_lazy.collect().to_pandas()) + st.session_state["filtered_df"] = filtered_df + + # Layout response metrics and the filtered matrix + st.success(f"Successfully applied **{filter_method}** filter!") + + # Display dataset scale compression stats + col1, col2, col3 = st.columns(3) + col1.metric("Original Proteins", pivot_df.shape[0]) + col2.metric("Filtered Proteins", filtered_df.shape[0]) + col3.metric( + "Removed Proteins", pivot_df.shape[0] - filtered_df.shape[0], delta=None + ) + + st.subheader("Filtered Abundance Table") + if filtered_df.empty: + st.warning( + "The filtered table is empty. Try relaxing the threshold constraints." + ) + else: + st.dataframe(filtered_df, use_container_width=True) \ No newline at end of file diff --git a/content/imputation.py b/content/imputation.py new file mode 100644 index 0000000..4350263 --- /dev/null +++ b/content/imputation.py @@ -0,0 +1,145 @@ +"""Imputation Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import imputation algorithms from openms_insight engine +from openms_insight.analysis.imputation import impute_mar, impute_smallest_value + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Missing Value Imputation") + +st.markdown( + """ +Handle missing values (zeros or nulls) in your quantification matrix using biological group-aware (MAR) or absolute lowest limit (MNAR) techniques. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load base dataset and clean dictionary keys +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="📋" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Pipeline Checkpoint: Fetch upstream filtered data if available, fallback to raw pivot matrix +if "filtered_df" in st.session_state and st.session_state["filtered_df"] is not None: + base_df = strip_stat_columns(st.session_state["filtered_df"]) + st.session_state["filtered_df"] = base_df + st.info( + "🔄 **Upstream Pipeline Detected**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No filtering history found. Operating on the original unfiltered table." + ) + +# 2. Identify actual sample columns dynamically based on the current active matrix +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Input Matrix Summary --- +st.subheader("Input Matrix Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Imputation Configuration --- +st.subheader("Configure Imputation Engine") + +# Build Polars structural metadata DataFrame +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for core missingness assumption strategy +impute_category = st.selectbox( + "Select Imputation Class", + options=["MAR (Missing At Random)", "MNAR (Missing Not At Random)"], + index=0, + help="MAR uses group metrics (Mean/Median). MNAR shifts values below the limit of detection.", +) + +# Render algorithmic options sub-menus based on the parent selection +if impute_category == "MAR (Missing At Random)": + st.markdown( + "**Group Character Imputation**: Fills missing metrics leveraging sample properties belonging to the same group." + ) + strategy_opt = st.radio( + "Mathematical Strategy", + options=["median", "mean"], + index=0, + horizontal=True, + ) + +elif impute_category == "MNAR (Missing Not At Random)": + st.markdown( + "**Smallest Value Imputation**: Replaces missing items with the minimum values detected to reflect technical dropout limits." + ) + scope_opt = st.radio( + "Detection Minimum Scope", + options=["row", "global"], + index=0, + horizontal=True, + help="'row' targets current protein minimum; 'global' searches the entire mass spectrometry matrix profile.", + ) + +# --- SECTION 3: Imputation Execution --- +if st.button("Apply Imputation", type="primary"): + # Initialize optimization pipeline graph via lazy loading conversion + quant_lazy = pl.from_pandas(base_df).lazy() + + # Route configuration matrix parameters to designated engine function channels + if impute_category == "MAR (Missing At Random)": + imputed_lazy = impute_mar( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + strategy=strategy_opt, + ) + elif impute_category == "MNAR (Missing Not At Random)": + imputed_lazy = impute_smallest_value( + quantification_data=quant_lazy, metadata=metadata_pl, scope=scope_opt + ) + + # Resolve lazy graph optimization tree and push to display data frame structure + imputed_df = strip_stat_columns(imputed_lazy.collect().to_pandas()) + + # 💾 Save current output into Session State for down-stream processing (Normalization, Statistics) + st.session_state["imputed_df"] = imputed_df + + st.success(f"Successfully finalized **{impute_category}** imputation step!") + + # Calculate and display a quick performance matrix check + st.subheader("Imputed Result Table") + st.dataframe(imputed_df, use_container_width=True) \ No newline at end of file diff --git a/content/normalization.py b/content/normalization.py new file mode 100644 index 0000000..c0e97e8 --- /dev/null +++ b/content/normalization.py @@ -0,0 +1,242 @@ +"""Normalization Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import normalization engine functions from openms_insight +from openms_insight.analysis.normalization import ( + normalize_samples, + scale_data, + transform_data, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None: + """Keep preprocessing tables intensity-only before statistical analysis.""" + if df is None: + return None + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Normalization & Scaling") + +st.markdown( + """ +Standardize and transform your protein abundance profiles to correct for technical variations and optimize statistical distributions. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="📋" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +filtered_df = strip_stat_columns(st.session_state.get("filtered_df")) +imputed_df = strip_stat_columns(st.session_state.get("imputed_df")) +normalized_df = strip_stat_columns(st.session_state.get("normalized_df")) +if filtered_df is not None: + st.session_state["filtered_df"] = filtered_df +if imputed_df is not None: + st.session_state["imputed_df"] = imputed_df +if normalized_df is not None: + st.session_state["normalized_df"] = normalized_df + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = imputed_df + st.info( + "🔄 **Upstream Pipeline Detected**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = filtered_df + st.warning( + "⚠️ **Imputation Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original unfiltered table." + ) + +# 2. Extract actual active sample columns dynamically +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("### Pipeline Overview") +st.caption("Data flows in order: Filtering -> Imputation -> Normalization") + +step_rows = [ + { + "Step": "Filtering", + "Status": "Done" if filtered_df is not None else "Not run", + "Rows": filtered_df.shape[0] if filtered_df is not None else "-", + "Cols": filtered_df.shape[1] if filtered_df is not None else "-", + }, + { + "Step": "Imputation", + "Status": "Done" if imputed_df is not None else "Not run", + "Rows": imputed_df.shape[0] if imputed_df is not None else "-", + "Cols": imputed_df.shape[1] if imputed_df is not None else "-", + }, + { + "Step": "Normalization", + "Status": "Done" if normalized_df is not None else "Not run", + "Rows": normalized_df.shape[0] if normalized_df is not None else "-", + "Cols": normalized_df.shape[1] if normalized_df is not None else "-", + }, +] +st.dataframe(pd.DataFrame(step_rows), hide_index=True, use_container_width=True) + +with st.expander("Show step tables", expanded=False): + if filtered_df is not None: + st.markdown("#### Filtering output") + st.dataframe(filtered_df.head(10), use_container_width=True) + if imputed_df is not None: + st.markdown("#### Imputation output") + st.dataframe(imputed_df.head(10), use_container_width=True) + if normalized_df is not None: + st.markdown("#### Normalization output") + st.dataframe(normalized_df.head(10), use_container_width=True) + if filtered_df is None and imputed_df is None and normalized_df is None: + st.info("No preprocessing outputs yet. Start from Filtering.") + +st.markdown("---") + +# --- SECTION 2: Normalization Parameter Configuration --- +st.subheader("Configure Preprocessing & Scaling Chains") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2, col3 = st.columns(3) + +with col1: + st.markdown("### 🧬 1. Mathematical Transformation") + transform_strategy = st.selectbox( + "Select Transformation", + options=["None", "log2", "log10", "square_root", "cube_root"], + index=0, + help="Compress data dynamic range and stabilize heteroscedastic variance profiles.", + ) + +with col2: + st.markdown("### 🧪 2. Sample Normalization") + norm_strategy = st.selectbox( + "Select Normalization", + options=["None", "sum", "median", "pqn", "reference_feature", "quantile"], + index=0, + help="Perform column-wise corrections to account for variable sample loading concentrations.", + ) + + # Conditionally display target input field for reference feature matching + ref_feature_input = None + if norm_strategy == "reference_feature": + ref_feature_input = st.text_input( + "Reference Protein Name (ID)", + value="", + placeholder="e.g., P01234 or GAPDH", + help=f"Enter the exact unique identifier string matching a key inside the '{id_col}' column.", + ) + +with col3: + st.markdown("### 📊 3. Row Scaling") + scaling_strategy = st.selectbox( + "Select Scaling Mode", + options=["None", "mean_centering", "auto_scaling", "pareto_scaling", "range_scaling"], + index=0, + help="Adjust individual feature weights to make low and high abundance proteins comparable.", + ) + + +# --- SECTION 3: Normalization Pipe Sequential Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Apply Normalization Pipelines", type="primary"): + + # Validate reference feature selection if active before hitting polars execution layers + if norm_strategy == "reference_feature" and not ref_feature_input: + st.error( + "❌ Validation Error: Please provide a valid Reference Protein Name to use the 'reference_feature' strategy." + ) + st.stop() + + # Convert pandas memory buffer into optimization lazy dataframe tree graph + processing_lazy = pl.from_pandas(base_df).lazy() + + # Execute Chain 1: Transform Matrix Data + try: + processing_lazy = transform_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=transform_strategy, + ) + + # Execute Chain 2: Normalize Sample Intensities (Columns) + processing_lazy = normalize_samples( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=norm_strategy, + id_col=id_col, + reference_feature=ref_feature_input if norm_strategy == "reference_feature" else None, + ) + + # Execute Chain 3: Scale Individual Features (Rows) + processing_lazy = scale_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=scaling_strategy, + ) + + # Finalize and collect pipeline query graph optimizations + normalized_df = strip_stat_columns(processing_lazy.collect().to_pandas()) + + # 💾 Save processing checkpoint inside Session State for Downstream (Statistics Block) + st.session_state["normalized_df"] = normalized_df + + st.success("Successfully executed all selected normalization pipelines!") + + # Display the finalized transformation matrix view + st.subheader("Normalized Abundance Table") + st.dataframe(normalized_df, use_container_width=True) + + except ValueError as val_err: + # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein) + st.error(f"Engine Configuration Error: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/content/results_abundance.py b/content/results_abundance.py index a86f1a3..38c42bc 100644 --- a/content/results_abundance.py +++ b/content/results_abundance.py @@ -1,6 +1,7 @@ """Abundance (ProteomicsLFQ) Results Page.""" import streamlit as st import pandas as pd +import numpy as np from pathlib import Path from src.common.common import page_setup from src.common.results_helpers import get_workflow_dir, get_abundance_data @@ -12,7 +13,7 @@ st.markdown( """ View protein and PSM-level quantification from **ProteomicsLFQ**. -This page calculates differential expression statistics between sample groups. +This page focuses on raw abundance intensity for preprocessing. """ ) @@ -40,49 +41,44 @@ csv_file = csv_files[0] -def render_protein_table(pivot_df, group_map, is_lfq=True): +def render_protein_table(pivot_df, is_lfq=True): """Common function to render the protein-level abundance table""" + pivot_df = pivot_df.copy() st.markdown("### Protein-Level Abundance Table") st.info( "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." + "same protein and aggregating their intensities across samples." ) - # Display group comparison info - groups = sorted(set(group_map.values())) - if len(groups) >= 2: - group1, group2 = sorted(groups)[:2] - st.info(f"Statistical comparison: **{group2} vs {group1}**") - if is_lfq: # Handle LFQ mode columns (Raw Intensity) id_col = "ProteinName" - exclude_cols = [id_col, "log2FC", "p-value", "PeptideSequence"] + exclude_cols = [id_col, "PeptideSequence"] sample_cols = [c for c in pivot_df.columns if c not in exclude_cols] - + pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) - display_cols = [id_col, "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"] + display_cols = [id_col, "Intensity"] + sample_cols + ["PeptideSequence"] help_text = "Raw sample intensities" y_min = None else: # Handle non-LFQ mode columns (Log2-transformed Intensity) id_col = "protein" - exclude_cols = [id_col, "log2FC", "p-value", "p-adj", "n_proteins", "n_peptides", "protein_score"] + exclude_cols = [id_col, "n_proteins", "n_peptides", "protein_score"] sample_cols = [c for c in pivot_df.columns if c not in exclude_cols and "ratio" not in c.lower()] - + pivot_df["Intensity"] = pivot_df[sample_cols].apply( lambda row: [np.log2(v + 1) for v in row], axis=1 ) - display_cols = [id_col, "log2FC", "p-value", "Intensity"] + sample_cols + display_cols = [id_col, "Intensity"] + sample_cols help_text = "Sample intensities (log2 scale)" y_min = 0 # Filter to available columns, then sort and display available_cols = [c for c in display_cols if c in pivot_df.columns] - + view_df = pivot_df[available_cols] + st.dataframe( - pivot_df[available_cols].sort_values("p-value"), + view_df, column_config={ "Intensity": st.column_config.BarChartColumn( "Intensity", @@ -110,12 +106,11 @@ def render_protein_table(pivot_df, group_map, is_lfq=True): with protein_tab: if result is None: - st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.") - # st.page_link("content/workflow_configure.py", label="Go to Configure", icon="⚙️") + st.warning("Could not load abundance data. Please run the workflow first.") st.stop() pivot_df, expr_df, group_map = result - render_protein_table(pivot_df, group_map, is_lfq=True) + render_protein_table(pivot_df, is_lfq=True) with psm_tab: st.markdown("### PSM-level Quantification Table") @@ -130,27 +125,27 @@ def render_protein_table(pivot_df, group_map, is_lfq=True): pre_processing_tab, protein_tab = st.tabs(["Pre-processing", "Protein Table"]) if result is None: - st.info("💡 Please complete the configuration in the 'Configure' page to see results.") + st.info("💡 Please run the workflow first to see results.") st.stop() - + pivot_df, expr_df, group_map = result with pre_processing_tab: - st.write("### Final Results (Group row removed, Stats added)") + st.write("### Final Results (Intensity matrix)") st.dataframe(pivot_df.head(10)) with protein_tab: - render_protein_table(pivot_df, group_map, is_lfq=False) + render_protein_table(pivot_df, is_lfq=False) except Exception as e: st.error(f"Failed to load {csv_file.name}: {e}") st.markdown("---") -st.markdown("**Next steps:** Explore statistical visualizations") +st.markdown("**Next steps:** Continue preprocessing, then run statistical inference") col1, col2, col3 = st.columns(3) with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="🌋") + st.page_link("content/filtering.py", label="Filtering", icon="🧹") with col2: - st.page_link("content/results_pca.py", label="PCA", icon="📊") + st.page_link("content/imputation.py", label="Imputation", icon="🧩") with col3: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="🔥") + st.page_link("content/statistical.py", label="Statistical Inference", icon="🔬") \ No newline at end of file diff --git a/content/results_heatmap.py b/content/results_heatmap.py index 72b8438..104bff6 100644 --- a/content/results_heatmap.py +++ b/content/results_heatmap.py @@ -1,20 +1,18 @@ """Heatmap Results Page.""" import streamlit as st import numpy as np -import plotly.express as px -from scipy.cluster.hierarchy import linkage, leaves_list -from scipy.spatial.distance import pdist +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data, get_workflow_dir -from src.workflow.ParameterManager import ParameterManager +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import Heatmap params = page_setup() st.title("Heatmap") st.markdown( """ -Hierarchically clustered heatmap of protein-level abundance (Z-score normalized). -Proteins and samples are ordered by similarity. +Interactive hierarchically clustered heatmap of protein-level abundance (Z-score normalized). +Powered by OpenMS-Insight multi-resolution engine. """ ) @@ -29,104 +27,75 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) -workflow_dir = get_workflow_dir(st.session_state["workspace"]) -parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - -workflow_params = parameter_manager.get_parameters_from_json() -analysis_mode = workflow_params.get("analysis-mode", "LFQ") - -st.write("Workflow Analysis Mode:", analysis_mode) - -if analysis_mode == "LFQ": - top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") - - var_series = expr_df.var(axis=1) - top_proteins = var_series.sort_values(ascending=False).head(top_n).index - heatmap_df = expr_df.loc[top_proteins] - heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) - heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() - - if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) - - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) - - heatmap_clustered = heatmap_z.iloc[row_order, col_order] - - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) - - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} - ) - - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, use_container_width=True) - else: - st.warning("Insufficient data to generate the heatmap.") +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="🌋") - with col2: - st.page_link("content/results_pca.py", label="PCA", icon="📊") +sample_cols = expr_df.columns.tolist() + +# UI settings (number of top variance proteins) +top_n = st.slider("Number of proteins (Highest Variance)", 20, 200, 50, key="heatmap_top_n") + +# Process data (variance selection -> Z-score normalization) +var_series = expr_df.var(axis=1) +top_proteins = var_series.sort_values(ascending=False).head(top_n).index +heatmap_df = expr_df.loc[top_proteins] + +# Compute Z-scores and clean missing/invalid values +heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) +heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() + +if not heatmap_z.empty: + # Melt and convert data to Polars to satisfy OpenMS-Insight component requirements + # Restore the id column from the index as a regular column + heatmap_z_reset = heatmap_z.reset_index() + + # Unpivot the wide-format matrix into long-format (X, Y, Intensity) + melted_df = heatmap_z_reset.melt( + id_vars=[id_col], + value_vars=sample_cols, + var_name="Sample", + value_name="Z_score" + ) + + # Add sample group mapping if available for heatmap categories + if sample_group_map: + melted_df["Group"] = melted_df["Sample"].map(sample_group_map) + + # Pack the Pandas DataFrame into a Polars LazyFrame + heatmap_pl_lazy = pl.from_pandas(melted_df).lazy() + + # Initialize the OpenMS-Insight Heatmap component and map attributes + heatmap_component = Heatmap( + cache_id="quantms_protein_heatmap", + x_column="Sample", + y_column=id_col, + data=heatmap_pl_lazy, + intensity_column="Z_score", + title="Protein Abundance Heatmap (Z-score)", + x_label="Samples", + y_label="Proteins", + colorscale="RdBu", + reversescale=True, + log_scale=False, # Z-score can be negative, so log scale must stay off + intensity_label="Z-score", + category_column=None, + min_points=10000, # Generous point-count ceiling so the full grid renders + ) + + # Render the component + state_manager = st.session_state.get("state") + heatmap_component(state_manager=state_manager) else: - top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") - - var_series = expr_df.var(axis=1) - top_proteins = var_series.sort_values(ascending=False).head(top_n).index - heatmap_df = expr_df.loc[top_proteins] - heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) - heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() - - if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) - - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) - - heatmap_clustered = heatmap_z.iloc[row_order, col_order] - - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) - - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} - ) - - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, width="stretch") - else: - st.warning("Insufficient data to generate the heatmap.") - - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="🌋") - with col2: - st.page_link("content/results_pca.py", label="PCA", icon="📊") + st.warning("Insufficient data to generate the heatmap.") + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_volcano.py", label="Volcano Plot", icon="🌋") +with col2: + st.page_link("content/results_pca.py", label="PCA", icon="📊") diff --git a/content/results_heatmap_clustered.py b/content/results_heatmap_clustered.py new file mode 100644 index 0000000..7104c3a --- /dev/null +++ b/content/results_heatmap_clustered.py @@ -0,0 +1,107 @@ +"""Clustered Heatmap Results Page.""" +import streamlit as st +import numpy as np +import polars as pl +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import ClusteredHeatmap + +params = page_setup() +st.title("Clustered Heatmap") + +st.markdown( + """ +A real grid heatmap (rows = proteins, columns = samples) with hierarchical +clustering dendrograms on both axes and a sample-group color bar, powered +by OpenMS-Insight. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="📋") + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +top_n = st.slider("Number of proteins (Highest Variance)", 10, 200, 30, key="clustered_heatmap_top_n") + +var_series = expr_df.var(axis=1) +top_proteins = var_series.sort_values(ascending=False).head(top_n).index +heatmap_df = expr_df.loc[top_proteins] + +heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) +heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() + +if heatmap_z.empty: + st.warning("Insufficient data to generate the heatmap.") + st.stop() + +heatmap_z_reset = heatmap_z.reset_index() +heatmap_lazy = pl.from_pandas(heatmap_z_reset).lazy() + +sample_cols = expr_df.columns.tolist() +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, +) + +# Assign group annotation-bar colors in sorted-group order (matching how +# ClusteredHeatmap._preprocess() orders unique groups internally). +group_palette = [ + "#00BFC4", # teal + "#F8766D", # salmon + "#7CAE00", # yellow-green + "#C77CFF", # lavender purple + "#E7B800", # gold/amber + "#619CFF", # blue + "#FF61C3", # pink/magenta + "#00BA38", # green + "#FF8C42", # orange + "#00B0F6", # sky blue +] +unique_groups = sorted(set(sample_group_map.values())) +group_colors = {g: group_palette[i % len(group_palette)] for i, g in enumerate(unique_groups)} + +heatmap_component = ClusteredHeatmap( + cache_id="quantms_clustered_heatmap", + cache_path=str(st.session_state["workspace"]), + id_col=id_col, + data=heatmap_lazy, + metadata=metadata_pl, + row_cluster=True, + col_cluster=True, + title="Protein Abundance Heatmap (Z-score, clustered)", + x_label="Samples", + y_label="Proteins", + colorscale=[[0, "#6699E0"], [0.5, "#FFFFFF"], [1, "#E06666"]], + reversescale=False, + intensity_label="Z-score", + group_colors=group_colors, +) + +state_manager = st.session_state.get("state") +# Scale height with the number of proteins so row labels stay readable - +# BaseComponent otherwise defaults to a flat 400px, too short for a +# dendrogram+heatmap composite with more than a handful of rows. +heatmap_height = max(600, min(1400, 300 + top_n * 20)) +heatmap_component(state_manager=state_manager, height=heatmap_height) + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_volcano.py", label="Volcano Plot", icon="🌋") +with col2: + st.page_link("content/results_heatmap.py", label="Heatmap (original)", icon="🔥") diff --git a/content/results_pca.py b/content/results_pca.py index 466e475..29f441a 100644 --- a/content/results_pca.py +++ b/content/results_pca.py @@ -1,12 +1,10 @@ """PCA Results Page.""" -import streamlit as st import pandas as pd -import plotly.express as px -from sklearn.decomposition import PCA -from sklearn.preprocessing import StandardScaler +import polars as pl +import streamlit as st from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data, get_workflow_dir -from src.workflow.ParameterManager import ParameterManager +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import PCAPlot params = page_setup() st.title("PCA Analysis") @@ -14,7 +12,7 @@ st.markdown( """ Principal Component Analysis (PCA) of protein-level abundance. -Samples are colored by group assignment to visualize clustering. +Samples are projected onto their principal components and colored by group assignment to visualize clustering. """ ) @@ -22,6 +20,7 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Load abundance data (base wide-format table + sample -> group mapping) result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -29,88 +28,143 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +# Mirrors statistical.py: PCA should run on the most-processed data available. +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "🔄 **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-adj", "stat", "p-value"] +] +unique_groups = sorted({sample_group_map[s] for s in sample_cols if s in sample_group_map}) -workflow_dir = get_workflow_dir(st.session_state["workspace"]) -parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") -workflow_params = parameter_manager.get_parameters_from_json() -analysis_mode = workflow_params.get("analysis-mode", "LFQ") +if len(sample_cols) < 2: + st.info("PCA requires at least 2 samples.") + st.stop() -st.write("Workflow Analysis Mode:", analysis_mode) +if len(unique_groups) < 2: + st.warning( + "Only one biological group was detected - points will still be plotted, " + "but group-based coloring requires 2 or more groups." + ) -top_n = 500 +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples " + f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})." +) +st.dataframe(base_df, use_container_width=True) -if analysis_mode == "LFQ": - protein_col = "ProteinName" -else: - protein_col = "protein" +st.markdown("---") -top_proteins = ( - pivot_df - .dropna(subset=["p-adj"]) - .sort_values("p-adj", ascending=True) - .head(top_n)[protein_col] -) +# --- SECTION 2: PCA Configuration --- +st.subheader("Configure PCA") -expr_df_pca = expr_df.loc[ - expr_df.index.intersection(top_proteins) -] +expr_df_wide = base_df.set_index(id_col)[sample_cols] +max_available = expr_df_wide.shape[0] + +if max_available <= 20: + top_n = max_available + st.caption(f"Using all {top_n} proteins for PCA (dataset too small for variance filtering).") +else: + top_n = st.slider( + "Number of proteins (Highest Variance)", + min_value=20, + max_value=min(5000, max_available), + value=min(500, max_available), + step=10, + key="pca_top_n", + help=( + "PCA is computed only on the N proteins with the highest variance " + "across samples, to reduce noise from low-variance/uninformative features." + ), + ) + +top_proteins = expr_df_wide.var(axis=1).sort_values(ascending=False).head(top_n).index +expr_df_pca = expr_df_wide.loc[top_proteins].reset_index() if expr_df_pca.shape[0] < 2: - st.info("Not enough proteins after p-value filtering for PCA.") + st.info("Not enough proteins after variance filtering for PCA.") st.stop() -X = expr_df_pca.T -X_scaled = StandardScaler().fit_transform(X) - -pca = PCA(n_components=2) -pcs = pca.fit_transform(X_scaled) - -pca_df = pd.DataFrame( - pcs, - columns=["PC1", "PC2"], - index=X.index +# Prepare structural Polars metadata DataFrame required by PCAPlot +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, ) +pca_lazy = pl.from_pandas(expr_df_pca).lazy() + +# 3. Initialize the OpenMS-Insight PCAPlot component (computes PCA internally) +try: + pca_component = PCAPlot( + cache_id="quantms_pca_plot", + data=pca_lazy, + metadata=metadata_pl, + sample_id_field="sample_id", + group_field="group", + n_components=5, + title="Sample PCA", + ) +except ValueError as e: + st.error(f"PCA computation failed: {e}") + st.stop() -if analysis_mode == "LFQ": - norm_map = { - k.replace(".mzML", ""): v - for k, v in group_map.items() - } -else: - actual_sample_names = pca_df.index.tolist() - norm_map = {} - for k, v in group_map.items(): - try: - sample_idx = int(k) + 1 - target_substring = f"sample{sample_idx}[" - real_full_name = next((name for name in actual_sample_names if target_substring in name), None) - - if real_full_name: - norm_map[real_full_name] = v if v and v.strip() else "Unassigned" - except ValueError: - continue - -pca_df["Group"] = pca_df.index.map(norm_map) - -fig_pca = px.scatter( - pca_df, - x="PC1", - y="PC2", - color="Group", - text=pca_df.index, -) +variance_ratio = pca_component.get_variance_ratio() +pc_columns = pca_component.get_pc_columns() -fig_pca.update_traces(textposition="top center") -fig_pca.update_layout( - xaxis_title=f"PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)", - yaxis_title=f"PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)", - height=600, -) +# 4. Let the user pick which component pair to view (no recomputation needed) +col1, col2 = st.columns(2) +with col1: + pc_x_label = st.selectbox("X-axis component", pc_columns, index=0, key="pca_pc_x") +with col2: + default_y_index = 1 if len(pc_columns) > 1 else 0 + pc_y_label = st.selectbox("Y-axis component", pc_columns, index=default_y_index, key="pca_pc_y") -st.plotly_chart(fig_pca, width="stretch") +pc_x = int(pc_x_label.replace("PC", "")) +pc_y = int(pc_y_label.replace("PC", "")) -st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") -st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") +# 5. Render the component +state_manager = st.session_state.get("state") +pca_component(state_manager=state_manager, pc_x=pc_x, pc_y=pc_y, height=600) + +st.markdown( + "**Explained variance:** " + + ", ".join(f"{col} {ratio * 100:.1f}%" for col, ratio in zip(pc_columns, variance_ratio)) +) +st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by variance)") st.markdown("---") st.markdown("**Other visualizations:**") @@ -118,4 +172,4 @@ with col1: st.page_link("content/results_volcano.py", label="Volcano Plot", icon="🌋") with col2: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="🔥") \ No newline at end of file + st.page_link("content/results_heatmap.py", label="Heatmap", icon="🔥") diff --git a/content/results_proteomicslfq.py b/content/results_proteomicslfq.py index 77eb332..fde2ab9 100644 --- a/content/results_proteomicslfq.py +++ b/content/results_proteomicslfq.py @@ -45,15 +45,14 @@ st.markdown("### 🧬 Protein-Level Abundance Table") st.info( "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." + "same protein and aggregating their intensities across samples." ) if pivot_df.empty: st.info("No protein-level data available.") else: st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df.sort_values("p-value"), use_container_width=True) + st.dataframe(pivot_df, use_container_width=True) # ====================================================== # GO Enrichment Results diff --git a/content/results_volcano.py b/content/results_volcano.py index 5cc9b29..db2702f 100644 --- a/content/results_volcano.py +++ b/content/results_volcano.py @@ -1,10 +1,9 @@ """Volcano Plot Results Page.""" import streamlit as st -import plotly.express as px -import numpy as np +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data, get_workflow_dir -from src.workflow.ParameterManager import ParameterManager +from src.common.results_helpers import get_abundance_data, get_id_column +from openms_insight import VolcanoPlot params = page_setup() st.title("Volcano Plot") @@ -20,6 +19,19 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Check if statistical analysis results are available in the session state +if "statistics_df" not in st.session_state or st.session_state["statistics_df"] is None: + st.info("Statistical analysis data not found. Please run the statistical engine first.") + st.page_link("content/statistical.py", label="Go to Statistical Inference", icon="🔬") + st.stop() + +# Retrieve the completed statistical analysis DataFrame +statistics_df = st.session_state["statistics_df"] + +if statistics_df.empty: + st.info("No data available for volcano plot.") + st.stop() + result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -27,166 +39,63 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) + +# 2. Clean data and convert to Polars for component input +volcano_df = statistics_df.dropna(subset=["log2FC", "p-adj"]).copy() +volcano_pl_lazy = pl.from_pandas(volcano_df).lazy() + +# 3. Configure UI sliders (changing thresholds does not invalidate cache) +fc_thresh = st.slider( + "log2 Fold Change threshold", + min_value=0.5, + max_value=3.0, + value=1.0, + step=0.1, +) + +p_thresh = st.slider( + "p-adj (FDR) threshold", + min_value=0.001, + max_value=0.1, + value=0.05, + step=0.001, +) + +# 4. Initialize the OpenMS-Insight VolcanoPlot component +volcano_plot_component = VolcanoPlot( + cache_id="quantms_volcano_plot", + data=volcano_pl_lazy, + log2fc_column="log2FC", + pvalue_column="p-adj", + label_column=id_col, + up_color="#E74C3C", + down_color="#3498DB", + ns_color="#95A5A6", + show_threshold_lines=True, + threshold_line_style="dash", +) + +# 5. Render the component +state_manager = st.session_state.get("state") # Inject the project state management object + +volcano_plot_component( + state_manager=state_manager, + fc_threshold=fc_thresh, + p_threshold=p_thresh, + max_labels=10, # Display labels for the top N significant proteins + height=600, +) -if pivot_df.empty: - st.info("No data available for volcano plot.") - st.stop() - -workflow_dir = get_workflow_dir(st.session_state["workspace"]) -parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - -workflow_params = parameter_manager.get_parameters_from_json() -analysis_mode = workflow_params.get("analysis-mode", "LFQ") - -st.write("Workflow Analysis Mode:", analysis_mode) - -if analysis_mode == "LFQ": - volcano_df = pivot_df.copy() - volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"]) - - volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) - - fc_thresh = st.slider( - "log2 Fold Change threshold", - min_value=0.5, - max_value=3.0, - value=1.0, - step=0.1, - ) - - p_thresh = st.slider( - "p-adj (FDR) threshold", - min_value=0.001, - max_value=0.1, - value=0.05, - step=0.001, - ) - - volcano_df["Significance"] = "Not significant" - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", - ] = "Up-regulated" - - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", - ] = "Down-regulated" - - fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["ProteinName", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } - ) - - fig_volcano.add_vline(x=fc_thresh, line_dash="dash") - fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") - fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - - # Make x-axis symmetric around zero - max_abs_fc = volcano_df["log2FC"].abs().max() - x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding - - fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, - height=600, - ) - - st.plotly_chart(fig_volcano, use_container_width=True) - - up_count = (volcano_df["Significance"] == "Up-regulated").sum() - down_count = (volcano_df["Significance"] == "Down-regulated").sum() - st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") - - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_pca.py", label="PCA", icon="📊") - with col2: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="🔥") -else: - # Threshold Selection UI - st.divider() - c1, c2 = st.columns(2) - with c1: - fc_thresh = st.slider( - "log2 Fold Change threshold", - min_value=0.1, - max_value=3.0, - value=1.0, - step=0.1, - ) - with c2: - p_thresh = st.slider( - "p-adj (FDR) threshold", - min_value=0.001, - max_value=0.1, - value=0.05, - step=0.001, - ) - - volcano_df = pivot_df.dropna(subset=["log2FC", "p-adj"]).copy() - volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) - - volcano_df["Significance"] = "Not significant" - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", - ] = "Up-regulated" - - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", - ] = "Down-regulated" - - fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["protein", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } - ) - - fig_volcano.add_vline(x=fc_thresh, line_dash="dash") - fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") - fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - - # Make x-axis symmetric around zero - max_abs_fc = volcano_df["log2FC"].abs().max() - x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding - - fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, - height=600, - ) - - st.plotly_chart(fig_volcano, width="stretch") - - up_count = (volcano_df["Significance"] == "Up-regulated").sum() - down_count = (volcano_df["Significance"] == "Down-regulated").sum() - st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") - - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_pca.py", label="PCA", icon="📊") - with col2: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="🔥") +# 6. Keep the existing statistical summary and bottom links +up_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh)).sum() +down_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh)).sum() +st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_pca.py", label="PCA", icon="📊") +with col2: + st.page_link("content/results_heatmap.py", label="Heatmap", icon="🔥") diff --git a/content/statistical.py b/content/statistical.py new file mode 100644 index 0000000..2e2a46d --- /dev/null +++ b/content/statistical.py @@ -0,0 +1,165 @@ +"""Statistical Inference Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import statistics engine functions from openms_insight +from openms_insight.analysis.statistics import calculate_statistical_tests, adjust_fdr_lazy + +params = page_setup() +st.title("Statistical Inference") + +st.markdown( + """ +Run differential expression analysis to identify statistically significant proteins across your biological groups. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="📋" + ) + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "🔄 **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract actual active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] +unique_groups = sorted(list(set([sample_group_map[s] for s in sample_cols if s in sample_group_map]))) +group_count = len(unique_groups) + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Dynamic Statistical Parameter Configuration --- +st.subheader("Configure Statistical Engine") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2 = st.columns(2) + +with col1: + st.markdown("### 🔬 1. Hypothesis Testing Method") + + # Route available method options dynamically based on the group count + if group_count == 2: + method_options = ["limma_like", "welch", "paired"] + help_text = "'limma_like' uses Empirical Bayes variance shrinking. 'welch' is for unequal variances. 'paired' is for dependent samples." + elif group_count >= 3: + method_options = ["limma_like", "anova"] + help_text = "'limma_like' supports multi-group design matrices. 'anova' computes standard row-wise One-way ANOVA." + else: + st.error("❌ Statistical testing requires at least 2 unique sample groups.") + st.stop() + + selected_method = st.selectbox( + "Select Statistical Test", + options=method_options, + index=0, + help=help_text + ) + +with col2: + st.markdown("### 🛡️ 2. Multiple Testing Correction (FDR)") + selected_fdr = st.selectbox( + "Select FDR Adjustment Strategy", + options=["BH", "Bonferroni", "None"], + index=0, + help="'BH' (Benjamini-Hochberg) controls False Discovery Rate. 'Bonferroni' is strict Family-Wise Error Rate control." + ) + +# --- SECTION 3: Statistical Query Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Run Statistical Analysis", type="primary"): + + # Convert active pandas dataframe into polars lazyframe graph + stats_lazy = pl.from_pandas(base_df).lazy() + + try: + # Execute Chain 1: Calculate core statistics (Adds log2FC, stat, p-value) + stats_lazy = calculate_statistical_tests( + quantification_data=stats_lazy, + metadata=metadata_pl, + method=selected_method + ) + + # Execute Chain 2: Adjust Multiple Testing (Adds p-adj) + stats_lazy = adjust_fdr_lazy( + quantification_data=stats_lazy, + strategy=selected_fdr + ) + + # Resolve lazy graph optimization tree and bring back to pandas memory + statistics_df = stats_lazy.collect().to_pandas() + + # 💾 Save processing checkpoint inside Session State for Downstream (e.g., Volcano plot, Volcano/Heatmap UI) + st.session_state["statistics_df"] = statistics_df + + st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!") + + # Display the finalized statistics table view + st.subheader("Statistical Analysis Results") + st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`") + st.dataframe(statistics_df, use_container_width=True) + + except ValueError as val_err: + st.error(f"Engine Validation Fallure: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index e0a3e1c..889529e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,11 @@ services: - 8501:8501 volumes: - workspaces-streamlit-template:/workspaces-streamlit-template + # Optional: bind-mount a host directory of MS data files at the path + # that `local_data_dir` in settings.json points to (the Docker image + # defaults this to /mounted-data). When the directory exists at + # runtime, the upload page shows an in-app file browser for it. + # - /path/on/host:/mounted-data:ro environment: # Number of Streamlit server instances (default: 1 = no load balancer). # Set to >1 to enable nginx load balancing across multiple Streamlit instances. diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..65cbf8c --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,224 @@ +#!/bin/bash +# Container entrypoint for the OpenMS streamlit template. +# +# Works with both Docker (writable root FS, runs as root) and +# Apptainer/Singularity (read-only root FS, runs as the host user's UID). +# On HPC clusters apptainer is the dominant runtime; this script makes the +# image usable there without --writable-tmpfs. +set -e + +# Force the app directory regardless of how the container was invoked. +# `apptainer instance start` does not always honor the Docker WORKDIR, so +# `streamlit run app.py` would otherwise resolve against the host's CWD. +cd /app + +# Breadcrumbs — surfaced via apptainer instance .out/.err on failure, harmless +# in docker mode. Cheap to keep around for ongoing apptainer support. +echo "entrypoint: uid=$(id -u) gid=$(id -g) cwd=$(pwd) host=$(hostname) tty=$(tty 2>/dev/null || echo none)" +echo "entrypoint: APPTAINER_NAME=${APPTAINER_NAME:-unset} SINGULARITY_NAME=${SINGULARITY_NAME:-unset} APPTAINER_CONTAINER=${APPTAINER_CONTAINER:-unset}" + +source /root/miniforge3/bin/activate streamlit-env +echo "entrypoint: conda env activated, streamlit=$(command -v streamlit || echo NOT_FOUND)" + +# ----------------------------------------------------------------------------- +# Apptainer / read-only root filesystem detection +# ----------------------------------------------------------------------------- +# Apptainer sets APPTAINER_NAME (and SINGULARITY_NAME for backwards compat). +# As a fallback we probe /var/run for writability: docker = writable, apptainer +# default = read-only. Either signal flips us into "read-only mode". +if [ -n "${APPTAINER_NAME:-}" ] || [ -n "${SINGULARITY_NAME:-}" ] \ + || [ -n "${APPTAINER_CONTAINER:-}" ] || [ -n "${SINGULARITY_CONTAINER:-}" ] \ + || ! ( : > /var/run/.openms-rw-probe ) 2>/dev/null; then + READONLY_ROOT=1 + echo "Detected read-only root filesystem (apptainer/singularity mode)" +else + READONLY_ROOT=0 + rm -f /var/run/.openms-rw-probe 2>/dev/null || true +fi + +# Pick state paths. In read-only mode we must use /tmp (always tmpfs in +# apptainer); in docker mode we keep the conventional /var paths so existing +# docker-compose / k8s deployments are unaffected. +if [ "$READONLY_ROOT" -eq 1 ]; then + RUNTIME_DIR="${OPENMS_RUNTIME_DIR:-/tmp/openms-runtime-$$}" + mkdir -p "$RUNTIME_DIR" + REDIS_DATA_DIR="$RUNTIME_DIR/redis" + REDIS_PID_FILE="$RUNTIME_DIR/redis.pid" + # Apptainer/singularity share the host's network namespace by default. If + # the host has anything listening on 6379 (a system redis-server, a docker + # container, a previous singularity instance that didn't clean up), our + # `redis-server --daemonize` silently fails with EADDRINUSE and the local + # redis-cli ping happily connects to the host's redis instead — which + # leaves stale `worker-1` records lying around and ultimately runs the + # workflow's mkdir outside our mount namespace (no bind → EROFS). A unix + # socket sidesteps the network stack entirely; the path is unambiguously + # ours. + REDIS_SOCKET="$RUNTIME_DIR/redis.sock" + REDIS_URL="unix://${REDIS_SOCKET}" + export REDIS_URL + NGINX_CONF_DIR="$RUNTIME_DIR/nginx" + NGINX_PID_FILE="$RUNTIME_DIR/nginx.pid" + mkdir -p "$REDIS_DATA_DIR" "$NGINX_CONF_DIR" + # Marker for out-of-band discovery (e.g. `apptainer exec ... redis-cli` + # from CI). The entrypoint's exported env doesn't propagate to fresh + # exec invocations, so write the resolved URL to a stable path. + echo "$REDIS_URL" > /tmp/openms-redis-url 2>/dev/null || true +else + RUNTIME_DIR="/var/run" + REDIS_DATA_DIR="/var/lib/redis" + REDIS_PID_FILE="/var/run/redis.pid" + REDIS_SOCKET="" + NGINX_CONF_DIR="/etc/nginx" + NGINX_PID_FILE="/run/nginx.pid" +fi + +# ----------------------------------------------------------------------------- +# Workspace cleanup cron (best-effort) +# ----------------------------------------------------------------------------- +# `service cron start` writes /var/run/crond.pid; it cannot work on a read-only +# root. The cleanup job is optional — workspaces just accumulate until the +# container is rebuilt, which is acceptable for HPC use cases where users +# manage their own workspace volumes. +if [ "$READONLY_ROOT" -eq 0 ]; then + service cron start || echo "WARN: cron failed to start; workspace cleanup disabled" +else + echo "Skipping cron (read-only root); run clean-up-workspaces.py manually if needed" +fi + +# ----------------------------------------------------------------------------- +# Redis + RQ workers (only present in the full image) +# ----------------------------------------------------------------------------- +# The simple image does not install redis-server. Skip the whole queue section +# when the binary is missing, so this entrypoint can be shared by both images. +if command -v redis-server >/dev/null 2>&1; then + if [ -n "$REDIS_SOCKET" ]; then + echo "Starting Redis server (data=$REDIS_DATA_DIR, socket=$REDIS_SOCKET)..." + # --port 0 disables the TCP listener entirely — we only accept the + # unix socket. This is the whole point of switching to a socket in + # apptainer mode: the host's network namespace (shared by default) + # cannot conflict with us, and there is no fall-through to a stray + # host redis-server. + redis-server --daemonize yes \ + --dir "$REDIS_DATA_DIR" \ + --pidfile "$REDIS_PID_FILE" \ + --unixsocket "$REDIS_SOCKET" \ + --unixsocketperm 700 \ + --port 0 \ + --appendonly no + REDIS_CLI_ARGS=(-s "$REDIS_SOCKET") + else + echo "Starting Redis server (data=$REDIS_DATA_DIR)..." + redis-server --daemonize yes \ + --dir "$REDIS_DATA_DIR" \ + --pidfile "$REDIS_PID_FILE" \ + --appendonly no + REDIS_CLI_ARGS=() + fi + + # Bounded wait so a broken redis-server (e.g. socket can't be created or + # an unexpected fork failure) fails the container fast instead of hanging + # forever and never serving /_stcore/health. + REDIS_STARTUP_RETRIES="${REDIS_STARTUP_RETRIES:-30}" + for i in $(seq 1 "$REDIS_STARTUP_RETRIES"); do + if redis-cli "${REDIS_CLI_ARGS[@]}" ping >/dev/null 2>&1; then + echo "Redis is ready" + break + fi + echo "Waiting for Redis... attempt $i/$REDIS_STARTUP_RETRIES" + sleep 1 + done + if ! redis-cli "${REDIS_CLI_ARGS[@]}" ping >/dev/null 2>&1; then + echo "ERROR: Redis failed to become ready within ${REDIS_STARTUP_RETRIES}s" >&2 + exit 1 + fi + + WORKER_COUNT="${RQ_WORKER_COUNT:-1}" + echo "Starting $WORKER_COUNT RQ worker(s)..." + for i in $(seq 1 "$WORKER_COUNT"); do + rq worker openms-workflows --url "$REDIS_URL" --name "worker-$i" & + done +fi + +# ----------------------------------------------------------------------------- +# Streamlit (single instance or behind nginx load balancer) +# ----------------------------------------------------------------------------- +SERVER_COUNT="${STREAMLIT_SERVER_COUNT:-1}" + +# Surface a misconfigured opt-in to load balancing — silently downgrading to a +# single instance has bitten users on the simple image variant where nginx +# isn't installed. +if [ "$SERVER_COUNT" -gt 1 ] && ! command -v nginx >/dev/null 2>&1; then + echo "WARN: STREAMLIT_SERVER_COUNT=$SERVER_COUNT requested but nginx is not installed (simple image?); falling back to a single instance" >&2 +fi + +if [ "$SERVER_COUNT" -gt 1 ] && command -v nginx >/dev/null 2>&1; then + echo "Starting $SERVER_COUNT Streamlit instances with nginx load balancer..." + + UPSTREAM_SERVERS="" + BASE_PORT=8510 + for i in $(seq 0 $((SERVER_COUNT - 1))); do + PORT=$((BASE_PORT + i)) + UPSTREAM_SERVERS="${UPSTREAM_SERVERS} server 127.0.0.1:${PORT}; +" + done + + NGINX_CONF_FILE="$NGINX_CONF_DIR/nginx.conf" + cat > "$NGINX_CONF_FILE" <` + ```python + workspaces_directory = Path("/workspaces-streamlit-template") + ``` +3. Update `README.md` accordingly + + +**Dockerfile-related** +1. Choose one of the Dockerfiles depending on your use case: + - `Dockerfile` builds OpenMS including TOPP tools + - `Dockerfile_simple` uses pyOpenMS only +2. Update the Dockerfile: + - with the `GITHUB_USER` owning the Streamlit app repository + - with the `GITHUB_REPO` name of the Streamlit app repository + - if your main page Python file is not called `app.py`, modify the following line + ```dockerfile + RUN echo "mamba run --no-capture-output -n streamlit-env streamlit run app.py" >> /app/entrypoint.sh + ``` +3. Update Python package dependency files: + - `requirements.txt` if using `Dockerfile_simple` + - `environment.yml` if using `Dockerfile` + +## How to build a workflow + +### Simple workflow using pyOpenMS + +Take a look at the example pages `Simple Workflow` or `Workflow with mzML files` for examples (on the *sidebar*). Put Streamlit logic inside the pages and call the functions with workflow logic from from the `src` directory (for our examples `src/simple_workflow.py` and `src/mzmlfileworkflow.py`). + +### Complex workflow using TOPP tools + +This template app features a module in `src/workflow` that allows for complex and long workflows to be built very efficiently. Check out the `TOPP Workflow Framework` page for more information (on the *sidebar*). + +For building **conditional parameter UI** (widgets that appear or disappear based on another parameter's value), see the *Reactive parameters* subsection of the `TOPP Workflow Framework` page's *Parameter Input* section. diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..86653d1 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,100 @@ +# OpenMS streamlit app deployment + +OpenMS streamlit apps can be deployed two ways: + +- **Kubernetes** — Kustomize manifests under `k8s/` with CI-built images pushed to GHCR. See the "Developers Guide: Kubernetes Deployment" page. +- **Docker Compose** — described below. Uses the external [OpenMS/streamlit-deployment](https://github.com/OpenMS/streamlit-deployment) repo to aggregate multiple apps as submodules. + +If you're using Claude Code, the `configure-app-settings` and `configure-docker-compose-deployment` skills automate the docker-compose path; `configure-app-settings` + `configure-k8s-deployment` automate the Kubernetes path. + +--- + +## Docker Compose + +Multiple streamlit apps based on the [OpenMS streamlit template](https://github.com/OpenMS/streamlit-template/) can be deployed together using docker compose. + +## Features + +- deploy all OpenMS apps at once +- user data (in workspaces) is stored in persistent docker volumes for each app + +## Requirements +- Docker Compose + +## Deployment (e.g., needed after one app changed) + +**1. Make sure submodules are up-to-data.** + +`git submodule init` + +`git submodule update` + +**2. Specify GitHub token (to download Windows executables).** + +> This is **important**! Omitting this step while result in all apps not having the option to download executables any more. + +Create a temporary `.env` file with your Github token. It should contain only one line: + +`GITHUB_TOKEN=` + +**3. Run docker-compose.** + +`docker-compose up --build -d` + +> Make sure to remove the `.env` file with your Github token after successful build + +## Add new app + +This will add your app as a submodule to the streamlit deployment repository. + +**1. Enable online mode in the apps settings.json.** + +**2. Fork and clone the [OpenMS streamlit deployment](https://github.com/OpenMS/streamlit-deployment) repository locally.** + +**3. Add your app as submodule. Make sure the app name is not used already.** + +`git submodule add ` + +**4. Initialize and update submodules.** + +`git submodule init` + +`git submodule update` + +**5. Add your app to `docker-compose.yml` file as a new service.** + +Copy the last service as a template. + +Check and update the following entries: + +- name of the service + - the name of the submodule +- build context + - the relative path to the submodule +- build dockerfile + - the correct Dockerfile +- image + - name of the docker image (typically the service name with underscores) +- ports + - chose an incremental host port number from the last service pointing to the streamlit port in docker container (8501) +- volumes + - update the names of the workspace directories, user data is stored outside of the docker container in a docker volume +- command + - update command with your main streamlit file + +**6. Test everything works locally.** + +Run docker-compose to launch all services. + +`docker-compose up --build -d` + +- there should be no errors building all services +- make sure all apps are accessible via their port from localhost +- test functionality of your app + +**7. Make a pull request with your changes to OpenMS/streamlit-deployment main branch.** + + + +# Other Architectures +In principle OpenMS runs on most processor architectures. The images are provided and tested for x86 but OpenMS can also be compiled on architectures like arm64. Please note that you might have to adjust the miniforge version according to the processor architecture. diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..06e0465 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,109 @@ +# Installation + +## Windows + +The app is available as pre-packaged Windows executable, including all dependencies. + +The windows executable is built by a GitHub action and can be downloaded [here](https://github.com/OpenMS/streamlit-template/actions/workflows/build-windows-executable-app.yaml). +Select the latest successfull run and download the zip file from the artifacts section, while signed in to GitHub. + +## Python + +Clone the [streamlit-template repository](https://github.com/OpenMS/streamlit-template). It includes files to install dependencies via pip or conda. + +## 💻 Run Locally + +To run the app locally: + +1. **Clone the repository** + ```bash + git clone https://github.com/OpenMS/streamlit-template.git + cd streamlit-template + ``` + +2. **Install dependencies** + + Make sure you can run ```pip``` commands. + + Install all dependencies with: + ```bash + pip install -r requirements.txt + ``` + +4. **Launch the app** + ```bash + streamlit run app.py + ``` + +> ⚠️ Note: The local version offers limited functionality. Features that depend on OpenMS are only available in the Docker setup. + + +## 🐳 Build with Docker + +This repository contains two Dockerfiles. + +1. `Dockerfile`: This Dockerfile builds all dependencies for the app including Python packages and the OpenMS TOPP tools. Recommended for more complex workflows where you want to use the OpenMS TOPP tools for instance with the **TOPP Workflow Framework**. +2. `Dockerfile_simple`: This Dockerfile builds only the Python packages. Recommended for simple apps using pyOpenMS only. + +1. **Install Docker** + + Install Docker from the [official Docker installation guide](https://docs.docker.com/engine/install/) + +
+ Click to expand + + ```bash + # Remove older Docker versions (if any) + for pkg in docker.io docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do sudo apt-get remove -y $pkg; done + ``` + +
+ +2. **Test Docker** + + Verify that Docker is working. + ```bash + docker run hello-world + ``` + When running this command, you should see a hello world message from Docker. + +3. **Clone the repository** + ```bash + git clone https://github.com/OpenMS/streamlit-template.git + cd streamlit-template + ``` + +4. **Specify GitHub token (to download Windows executables).** + + Create a temporary `.env` file with your Github token. + + It should contain only one line: + `GITHUB_TOKEN=` + + ℹ️ **Note:** This step is not strictly required, but skipping it will remove the option to download executables from the WebApp. + +3. **Build & Launch the App** + + To build and start the containers. + From the project root directory: + + ```bash + docker-compose up -d --build + ``` + At the end, you should see this: + ``` + [+] Running 2/2 + ✔ openms-streamlit-template Built + ✔ Container openms-streamlit-template Started + ``` + + To make sure server started successfully, run `docker compose ps`. You should see `Up` status: + ``` + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 4abe0603e521 openms_streamlit_template "/app/entrypoint.sh …" 7 minutes ago Up 7 minutes 0.0.0.0:8501->8501/tcp, :::8501->8501/tcp openms-streamlit-template + ``` + + To map the port to default streamlit port `8501` and launch. + + ``` + docker run -p 8505:8501 openms_streamlit_template diff --git a/docs/toppframework.py b/docs/toppframework.py new file mode 100644 index 0000000..a91b2be --- /dev/null +++ b/docs/toppframework.py @@ -0,0 +1,348 @@ +import streamlit as st +from src.Workflow import Workflow +from src.workflow.StreamlitUI import StreamlitUI +from src.workflow.FileManager import FileManager +from src.workflow.CommandExecutor import CommandExecutor +from src.workflow.ParameterManager import ParameterManager +from inspect import getsource + +def content(): + st.title("TOPP Workflow Framework Documentation") + + st.markdown( + """ +## Features + +- streamlined methods for uploading files, setting parameters, and executing workflows +- automatic parameter handling +- quickly build parameter interface for TOPP tools with all parameters from *ini* files +- automatically create a log file for each workflow run with stdout and stderr +- workflow output updates automatically in short intervalls +- user can leave the app and return to the running workflow at any time +- quickly build a workflow with multiple steps channelling files between steps +""" + ) + + st.markdown( + """ +## Quickstart + +This repository contains a module in `src/workflow` that provides a framework for building and running analysis workflows. + +The `WorkflowManager` class provides the core workflow logic. It uses the `Logger`, `FileManager`, `ParameterManager`, and `CommandExecutor` classes to setup a complete workflow logic. + +To build your own workflow edit the file `src/TOPPWorkflow.py`. Use any streamlit components such as tabs (as shown in example), columns, or even expanders to organize the helper functions for displaying file upload and parameter widgets. + +> 💡 Simply set a name for the workflow and overwrite the **`upload`**, **`configure`**, **`execution`** and **`results`** methods in your **`Workflow`** class. + +The file `content/6_TOPP-Workflow.py` displays the workflow content and can, but does not have to be modified. + +The `Workflow` class contains four important members, which you can use to build your own workflow: + +> **`self.params`:** dictionary of parameters stored in a JSON file in the workflow directory. Parameter handling is done automatically. Default values are defined in input widgets and non-default values are stored in the JSON file. + +> **`self.ui`:** object of type `StreamlitUI` contains helper functions for building the parameter and file upload widgets. + +> **`self.executor`:** object of type `CommandExecutor` can be used to run any command line tool alone or in parallel and includes a convenient method for running TOPP tools. + +> **`self.logger`:** object of type `Logger` to write any output to a log file during workflow execution. + +> **`self.file_manager`:** object of type `FileManager` to handle file types and creation of output directories. +""" + ) + + with st.expander("**Complete example for custom Workflow class**", expanded=False): + st.code(getsource(Workflow)) + + st.markdown( + """ +## File Upload + +All input files for the workflow will be stored within the workflow directory in the subdirectory `input-files` within it's own subdirectory for the file type. + +The subdirectory name will be determined by a **key** that is defined in the `self.ui.upload_widget` method. The uploaded files are available by the specific key for parameter input widgets and accessible while building the workflow. + +Calling this method will create a complete file upload page with the following components: + +- file uploader +- list of currently uploaded files with this key (or a warning if there are none) +- button to delete all files + +Fallback files(s) can be specified, which will be used if the user doesn't upload any files. This can be useful for example for database files where a default is provided. +""" + ) + + st.code(getsource(Workflow.upload)) + + st.info( + "💡 Use the same **key** for parameter widgets, to select which of the uploaded files to use for analysis." + ) + + with st.expander("**Code documentation:**", expanded=True): + st.help(StreamlitUI.upload_widget) + + st.markdown( + """ +## Parameter Input + +The parameter page is already pre-defined as a form with buttons to **save parameters** and **load defaults** and a toggle to show TOPP tool parameters marked as advanced. + +Generating parameter input widgets is done with the `self.ui.input` method for any parameter and the `self.ui.input_TOPP` method for TOPP tools. + +**1. Choose `self.ui.input_widget` for any parameter not-related to a TOPP tool or `self.ui.select_input_file` for any input file:** + +It takes the obligatory **key** parameter. The key is used to access the parameter value in the workflow parameters dictionary `self.params`. Default values do not need to be specified in a separate file. Instead they are determined from the widgets default value automatically. Widget types can be specified or automatically determined from **default** and **options** parameters. It's suggested to add a **help** text and other parameters for numerical input. + +Make sure to match the **key** of the upload widget when calling `self.ui.input_TOPP`. + +**2. Choose `self.ui.input_TOPP` to automatically generate complete input sections for a TOPP tool:** + +It takes the obligatory **topp_tool_name** parameter and generates input widgets for each parameter present in the **ini** file (automatically created) except for input and output file parameters. For all input file parameters a widget needs to be created with `self.ui.select_input_file` with an appropriate **key**. For TOPP tool parameters only non-default values are stored. + +**3. Choose `self.ui.input_python` to automatically generate complete input sections for a custom Python tool:** + +Takes the obligatory **script_file** argument. The default location for the Python script files is in `src/python-tools` (in this case the `.py` file extension is optional in the **script_file** argument), however, any other path can be specified as well. Parameters need to be specified in the Python script in the **DEFAULTS** variable with the mandatory **key** and **value** parameters. +""" + ) + + with st.expander( + "Options to use as dictionary keys for parameter definitions (see `src/python-tools/example.py` for an example)" + ): + st.markdown( + """ +**Mandatory** keys for each parameter +- *key:* a unique identifier +- *value:* the default value + +**Optional** keys for each parameter +- *name:* the name of the parameter +- *hide:* don't show the parameter in the parameter section (e.g. for **input/output files**) +- *options:* a list of valid options for the parameter +- *min:* the minimum value for the parameter (int and float) +- *max:* the maximum value for the parameter (int and float) +- *step_size:* the step size for the parameter (int and float) +- *help:* a description of the parameter +- *widget_type:* the type of widget to use for the parameter (default: auto) +- *advanced:* whether or not the parameter is advanced (default: False) +""" + ) + + st.code(getsource(Workflow.configure)) + st.info( + "💡 Access parameter widget values by their **key** in the `self.params` object, e.g. `self.params['mzML-files']` will give all selected mzML files." + ) + + st.markdown( + """ +### Reactive parameters (conditional UI) + +By default every parameter widget is rendered inside an isolated `st.fragment`: changing it reruns only that widget, which keeps large parameter sections fast but also means a widget **cannot** show or hide other widgets. Pass **`reactive=True`** to `self.ui.input_widget`, `self.ui.select_input_file` or `self.ui.input_TOPP` to render the widget directly in the parent `configure()` scope instead — a change then reruns `configure()`, so you can conditionally render downstream widgets based on the current value. + +Read the current value from `st.session_state` (**not** `self.params`, which is only loaded once per run and is stale within the same rerun) using the parameter-manager prefixes: `param_prefix` for `input_widget` / `select_input_file` keys, and `topp_param_prefix` for TOPP keys of the form `":1:"` (the `` matches the tool's ini, as shown for each widget in the panel). +""" + ) + st.code( + '''@st.fragment +def configure(self) -> None: + pm = self.parameter_manager + + # A custom widget that reveals more widgets when checked + self.ui.input_widget( + "advanced-mode", False, "Advanced mode", + widget_type="checkbox", reactive=True, + ) + if st.session_state.get(f"{pm.param_prefix}advanced-mode", False): + self.ui.input_widget("threads", 4, "Threads", widget_type="number") + + # A TOPP parameter driving conditional UI (e.g. TMT type -> channel count) + self.ui.input_TOPP("IsobaricAnalyzer", reactive=True) + iso_type = st.session_state.get(f"{pm.topp_param_prefix}IsobaricAnalyzer:1:type", "") + if iso_type.startswith("tmt"): + self.ui.input_widget("tmt-channels", 10, "TMT channels", widget_type="number")''' + ) + st.info( + "💡 `reactive=True` is opt-in per widget and defaults to `False`. On `input_TOPP` it un-isolates the whole tool panel, so **any** parameter change reruns `configure()` — leave it off unless another widget's visibility depends on a value in that panel." + ) + + with st.expander("**Code documentation**", expanded=True): + st.help(StreamlitUI.input_widget) + st.help(StreamlitUI.select_input_file) + st.help(StreamlitUI.input_TOPP) + st.help(StreamlitUI.input_python) + + st.markdown( + """ +## Parameter Presets + +Presets provide a way to offer users quick parameter configuration options for common analysis scenarios. + +### Enabling Presets + +1. Create a `presets.json` file at the repository root +2. Add preset definitions for your workflow (workflow name must be normalized: lowercase with hyphens) +3. Presets automatically appear in the parameter section via `self.ui.preset_buttons()` + +### presets.json Structure + +```json +{ + "your-workflow-name": { + "Preset Display Name": { + "_description": "Tooltip text for the button", + "TOPPToolName": { + "algorithm:param:path": value + }, + "_general": { + "custom_widget_key": value + } + } + } +} +``` + +### Key Points + +- **Workflow matching**: Workflow name is normalized (lowercase, hyphens for spaces). "TOPP Workflow" → "topp-workflow" +- **TOPP parameters**: Use the full parameter path (e.g., `algorithm:common:noise_threshold_int`) +- **General parameters**: Use `_general` for custom `input_widget` parameters +- **Descriptions**: Keys prefixed with `_` are metadata (not applied as parameters) +- **Opt-in feature**: No `presets.json` = no buttons shown (backward compatible) +""" + ) + + with st.expander("**Code documentation**", expanded=True): + st.help(StreamlitUI.preset_buttons) + st.help(ParameterManager.load_presets) + st.help(ParameterManager.apply_preset) + + st.markdown( + """ +## Building the Workflow + +Building the workflow involves **calling all (TOPP) tools** using **`self.executor`** with **input and output files** based on the **`FileManager`** class. For TOPP tools non-input-output parameters are handled automatically. Parameters for other processes and workflow logic can be accessed via widget keys (set on the parameter page) in the **`self.params`** dictionary. + +### FileManager + +The `FileManager` class serves as an interface for unified input and output files with useful functionality specific to building workflows, such as **setting a (new) file type** and **subdirectory in the workflows result directory**. + +Use the **`get_files`** method to get a list of all file paths as strings. + +Optionally set the following parameters modify the files: + +- **set_file_type** (str): set new file types and result subdirectory. +- **set_results_dir** (str): set a new subdirectory in the workflows result directory. +- **collect** (bool): collect all files into a single list. Will return a list with a single entry, which is a list of all files. Useful to pass to tools which can handle multiple input files at once. +""" + ) + + st.code( + """ +# Get all file paths as strings from self.param entry. +mzML_files = self.file_manager.get_files(self.params["mzML-files]) +# mzML_files = ['../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Control.mzML', '../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Treatment.mzML'] + +# Creating output files for a TOPP tool, setting a new file type and result subdirectory name. +feature_detection_out = self.file_manager.get_files(mzML_files, set_file_type="featureXML", set_results_dir="feature-detection") +# feature_detection_out = ['../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Control.featureXML', '../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Treatment.featureXML'] + +# Setting a name for the output directory automatically (useful if you never plan to access these files within the results page). +feature_detection_out = self.file_manager.get_files(mzML_files, set_file_type="featureXML", set_results_dir="auto") +# feature_detection_out = ['../workspaces-streamlit-template/default/topp-workflow/results/6DUd/Control.featureXML', '../workspaces-streamlit-template/default/topp-workflow/results/6DUd/Treatment.featureXML'] + +# Combining all mzML files to be passed to a TOPP tool in a single run. Using "collected" files as argument for self.file_manager.get_files will "un-collect" them. +mzML_files = self.file_manager.get_files(mzML_files, collect=True) +# mzML_files = [['../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Control.mzML', '../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Treatment.mzML']] + """ + ) + + with st.expander("**Code documentation**", expanded=True): + st.help(FileManager.get_files) + + st.markdown( + """ +### Running commands + +It is possible to execute any command line command using the **`self.executor`** object, either a single command or a list of commands in parallel. Furthermore a method to run TOPP tools is included. + +**1. Single command** + +The `self.executor.run_command` method takes a single command as input and optionally logs stdout and stderr to the workflow log (default True). +""" + ) + + st.code( + """ +self.executor.run_command(["command", "arg1", "arg2", ...]) +""" + ) + + st.markdown( + """ +**2. Run multiple commands in parallel** + +The `self.executor.run_multiple_commands` method takes a list of commands as inputs. + +**3. Run TOPP tools** + +The `self.executor.run_topp` method takes a TOPP tool name as input and a dictionary of input and output files as input. The **keys** need to match the actual input and output parameter names of the TOPP tool. The **values** should be of type `FileManager`. All other **non-default parameters (from input widgets)** will be passed to the TOPP tool automatically. + +Depending on the number of input files, the TOPP tool will be run either in parallel or in a single run (using **`FileManager.collect`**). +""" + ) + + st.info( + """💡 **Input and output file order** + +In many tools, a single input file is processed to produce a single output file. +When dealing with lists of input or output files, the convention is that +files are paired based on their order. For instance, the n-th input file is +assumed to correspond to the n-th output file, maintaining a structured +relationship between input and output data. +""" + ) + st.code( + """ +# e.g. FeatureFinderMetabo takes single input files +in_files = self.file_manager.get_files(["sample1.mzML", "sample2.mzML"]) +out_files = self.file_manager.get_files(in_files, set_file_type="featureXML", set_results_dir="feature-detection") + +# Run FeatureFinderMetabo tool with input and output files in parallel for each pair of input/output files. +self.executor.run_topp("FeatureFinderMetabo", input_output={"in": in_files, "out": out_files}) +# FeaturFinderMetabo -in sample1.mzML -out workspace-dir/results/feature-detection/sample1.featureXML +# FeaturFinderMetabo -in sample2.mzML -out workspace-dir/results/feature-detection/sample2.featureXML + +# Run SiriusExport tool with mutliple input and output files. +out = self.file_manager.get_files("sirius.ms", set_results_dir="sirius-export") +self.executor.run_topp("SiriusExport", {"in": self.file_manager.get_files(in_files, collect=True), + "in_featureinfo": self.file_manager.get_files(out_files, collect=True), + "out": out_se}) +# SiriusExport -in sample1.mzML sample2.mzML -in_featureinfo sample1.featureXML sample2.featureXML -out sirius.ms + """ + ) + + st.markdown( + """ +**4. Run custom Python scripts** + +Sometimes it is useful to run custom Python scripts, for example for extra functionality which is not included in a TOPP tool. + +`self.executor.run_python` works similar to `self.executor.run_topp`, but takes a single Python script as input instead of a TOPP tool name. The default location for the Python script files is in `src/python-tools` (in this case the `.py` file extension is optional in the **script_file** argument), however, any other path can be specified as well. Input and output file parameters need to be specified in the **input_output** dictionary. +""" + ) + + st.code( + """ +# e.g. example Python tool which modifies mzML files in place based on experimental design +self.ui.input_python(script_file="example", input_output={"in": in_mzML, "in_experimantal_design": FileManager(["path/to/experimantal-design.tsv"])}) + """ + ) + + st.markdown("**Example for a complete workflow:**") + + st.code(getsource(Workflow.execution)) + + with st.expander("**Code documentation**", expanded=True): + st.help(CommandExecutor.run_command) + st.help(CommandExecutor.run_multiple_commands) + st.help(CommandExecutor.run_topp) + st.help(CommandExecutor.run_python) \ No newline at end of file diff --git a/docs/user_guide.md b/docs/user_guide.md new file mode 100644 index 0000000..63bf521 --- /dev/null +++ b/docs/user_guide.md @@ -0,0 +1,77 @@ +# User Guide + +Welcome to the OpenMS Streamlit Web Application! This guide will help you understand how to use our tools effectively. + +## Advantages of OpenMS Web Apps + +OpenMS web applications provide a user-friendly interface for accessing the powerful features of OpenMS. Here are a few advantages: +- **Accessibility**: Access powerful OpenMS algorithms and TOPP tools from any device with a web browser. +- **Ease of Use**: Simplified user interface makes it easy for both beginners and experts to perform complex analyses. +- **No Installation Required**: Use the tools without the need to install OpenMS locally, saving time and system resources. + +## Workspaces + +In the OpenMS web application, workspaces are designed to keep your analysis organized: +- **Workspace Specific Parameters and Files**: Each workspace stores parameters and files (uploaded input files and results from workflows). +- **Persistence**: Your workspaces and parameters are saved, so you can return to your analysis anytime and pick up where you left off. Simply bookmark the page! + + +### File Uploads +- **Online Mode**: You can upload only one file at a time. This helps manage server load and optimizes performance. + +- **Local Mode**: Multiple file uploads are supported, giving you flexibility when working with large datasets. Additionally, the file size upload limit can be adjusted in the following ways: + 1. **Using `.streamlit/config.toml`**: + - You can modify the `.streamlit/config.toml` file and set the `maxUploadSize` parameter to your desired value. By default, this is set to 200MB. + - Example: + ```toml + [server] + maxUploadSize = 500 # Set the upload limit to 500MB + ``` + 2. **Using CLI Command**: + - You can customize the file size upload limit directly when running the application using the `--server.maxUploadSize` argument. + - Example: + ```bash + python run_app.py --server.maxUploadSize 500 + ``` + - This sets the upload limit to 500MB for the current session. + +- **Workspace Access**: + - In online mode, workspaces are stored temporarily and will be cleared after seven days of inactivity. + - In local mode, workspaces are saved on your local machine, allowing for persistent storage. Workspace directory can be specified in the `settings.json`. Defaults to `..` (parent directory). + +## Downloading Results + +You can download the results of your analyses, including data, figures and tables, directly from the application: +- **Figures**: Click the camera icon button, appearing while hovering on the top right corner of the figure. Set the desired image format in the settings panel in the side bar. +- **Tables**: Use the download button to save tables in *csv* format, appearing while hovering on the top right corner of the table. +- **Data**: Use the download section in the sidebar to download the raw results of your analysis. + +## Getting Started + +To get started: +1. Select or create a new workspace. +2. Upload your data file. +3. Set the necessary parameters for your analysis. +4. Run the analysis. +5. View and download your results. + +For more detailed information on each step, refer to the specific sections of this guide. + +## Parameter Presets + +Parameter presets allow you to quickly apply optimized parameter configurations for common analysis scenarios. When available, preset buttons appear on the parameter configuration page. + +### Using Presets + +1. Navigate to the parameter configuration page +2. Look for the **Parameter Presets** section below the "Show advanced parameters" toggle +3. Hover over a preset button to see its description +4. Click a preset to apply its optimized parameters +5. A confirmation message will appear when the preset is applied + +### What Presets Do + +- Presets override specific tool parameters with pre-configured values +- Only the parameters defined in the preset are changed; other parameters remain at their current values +- You can still modify individual parameters after applying a preset +- Use **Load default parameters** to reset all parameters to their original defaults \ No newline at end of file diff --git a/docs/win_exe_with_embed_py.md b/docs/win_exe_with_embed_py.md new file mode 100644 index 0000000..cb3d054 --- /dev/null +++ b/docs/win_exe_with_embed_py.md @@ -0,0 +1,278 @@ +## 💻 Create a window executable of a Streamlit App with embeddable Python + +To create an executable for Streamlit app on Windows, we'll use an embeddable version of Python.
+Here's a step-by-step guide: + +### Prerequisites + +You need a **system Python installation** (the regular Python installer from python.org) of the same version as the embeddable Python you'll download. This is required because the embeddable Python lacks development headers (`Python.h`) needed to compile native extensions. + +Install Python 3.11.9 from https://www.python.org/downloads/ if you don't have it already. + +### Download and Extract Python Embeddable Version + +1. Download a suitable Python embeddable version. For example, let's download Python 3.11.9: + + ```bash + # use curl command or manually download + curl -O https://www.python.org/ftp/python/3.11.9/python-3.11.9-embed-amd64.zip + ``` + +2. Extract the downloaded zip file: + + ```bash + mkdir python-3.11.9 + + unzip python-3.11.9-embed-amd64.zip -d python-3.11.9 + + rm python-3.11.9-embed-amd64.zip + ``` + +### Configure Python Environment + +1. Uncomment 'import site' in the `._pth` file: + + ```bash + # Uncomment to run site.main() automatically + # Remove hash from python-3.11.9/python311._pth file + import site + + # Or use command + sed -i '/^\s*#\s*import\s\+site/s/^#//' python-3.11.9/python311._pth + ``` + +### Install Required Packages + +Install all required packages from `requirements.txt` using the **system Python** with `--target` to install into the embeddable Python's site-packages: + +```bash +# Use system Python (which has development headers) to compile packages, +# installing into the embeddable Python's site-packages directory. +# The embeddable Python lacks Python.h headers needed for native extensions. +python -m pip install -r requirements.txt --target python-3.11.9/Lib/site-packages --upgrade --no-warn-script-location +``` + +> **Important**: Do NOT use `./python-3.11.9/python -m pip install ...` directly. The embeddable Python lacks the development headers required to compile native extensions (e.g., `Python.h`), which will cause builds to fail with errors like: +> ``` +> fatal error C1083: Cannot open include file: 'Python.h': No such file or directory +> ``` + +### Test and create `run_app.bat` file + +1. Test by running app + + ```batch + .\python-3.11.9\python -m streamlit run app.py + ``` + +2. Create a Clickable Shortcut + + Create a `run_app.bat` file to make running the app easier: + + ```batch + echo @echo off > run_app.bat + echo .\\python-3.11.9\\python -m streamlit run app.py >> run_app.bat + ``` + +### Create one executable folder + +1. Create a folder for your Streamlit app: + + ```bash + mkdir ../streamlit_exe + ``` + +2. Copy environment and app files: + + ```bash + # move Python environment folder + mv python-3.11.9 ../streamlit_exe + + # move run_app.bat file + mv run_app.bat ../streamlit_exe + + # copy streamlit app files + cp -r src pages .streamlit assets example-data ../streamlit_exe + cp app.py ../streamlit_exe + ``` + +3. Remove the server address from the bundled config to use `localhost` (default) instead of `0.0.0.0`, which doesn't work as a connect address on Windows: + + ```powershell + (Get-Content streamlit_exe/.streamlit/config.toml) -notmatch '^address' | Set-Content streamlit_exe/.streamlit/config.toml + ``` + +#### 🚀 After successfully completing all these steps, the Streamlit app will be available by running the run_app.bat file. + +:pencil: You can still change the configuration of Streamlit app with .streamlit/config.toml file, e.g., provide a different port, change upload size, etc. + +## Build executable in github action automatically + +Automate the process of building executables for your project with the GitHub action example [Test streamlit executable for Windows with embeddable python](https://github.com/OpenMS/streamlit-template/blob/main/.github/workflows/test-win-exe-w-embed-py.yaml) +
+ +## Create MSI Installer using WiX Toolset + +After creating your executable folder, you can package it into an MSI installer using WiX Toolset. Here's how: + +### 1. Set Environment Variables + +Set these variables for consistent naming throughout the process: + +```batch +APP_NAME=OpenMS-StreamlitTemplateApp +APP_UpgradeCode= generate-new +``` + +To generate a new GUID for your application's UpgradeCode, you can use: + +- PowerShell: `[guid]::NewGuid().ToString()` +- Online GUID generator: https://www.guidgen.com/ +- Windows Command Prompt: `powershell -Command "[guid]::NewGuid().ToString()"` + +### 2. Install WiX Toolset + +1. Download WiX Toolset binaries: + ```batch + curl -LO https://github.com/wixtoolset/wix3/releases/download/wix3111rtm/wix311-binaries.zip + unzip wix311-binaries.zip -d wix + ``` + +### 3. Prepare Installation Files + +1. Create a SourceDir structure: + + ```batch + mkdir SourceDir + move streamlit_exe\* SourceDir + ``` + +2. Create Readme.txt: + + ```batch + # Create a Readme.txt file in the SourceDir folder with instructions + # for launching the application + ``` + +3. Add necessary assets: + - Copy license file: `copy assets\openms_license.rtf SourceDir\` + - Copy app icon: `copy assets\openms.ico SourceDir\` + - Create success message script: + ```vbscript + ' ShowSuccessMessage.vbs + MsgBox "The " & "%APP_NAME%" & " application is successfully installed.", vbInformation, "Installation Complete" + ``` + +### 4. Generate WiX Source Files + +1. Generate component list from your files: + + ```batch + wix\heat.exe dir SourceDir -gg -sfrag -sreg -srd -template component -cg StreamlitExeFiles -dr AppSubFolder -out streamlit_exe_files.wxs + ``` + +2. Create main WiX configuration file (streamlit_exe.wxs): + + ```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + NOT Installed + + + + + + + ``` + +### 5. Build the MSI + +1. Compile WiX source files: + + ```batch + # Generate wixobj files from the WiX source files + wix\candle.exe streamlit_exe.wxs streamlit_exe_files.wxs + ``` + +2. Link and create MSI: + ```batch + # Create the MSI installer from the wixobj files + # The -sice:ICE60 flag stops a warning about duplicate component GUIDs, which can happen when heat.exe auto-generates components + wix\light.exe -ext WixUIExtension -sice:ICE60 -o %APP_NAME%.msi streamlit_exe_files.wixobj streamlit_exe.wixobj + ``` + +### 6. Additional Notes + +- The generated MSI will create desktop and start menu shortcuts +- Installation requires elevated privileges +- A success message will be shown after installation +- The installer includes a proper license agreement page +- All files will be installed in Program Files by default + +For more detailed customization options, refer to the [WiX Toolset documentation](https://wixtoolset.org/documentation/). + +:warning: The `APP_UpgradeCode` GUID should be unique for your application. Generate a new one if you're creating a different app. diff --git a/docs/win_exe_with_pyinstaller.md b/docs/win_exe_with_pyinstaller.md new file mode 100644 index 0000000..7112691 --- /dev/null +++ b/docs/win_exe_with_pyinstaller.md @@ -0,0 +1,140 @@ +## 💻 Create a window executable of streamlit app with pyinstaller +:heavy_check_mark: +Tested with streamlit v1.29.0, python v3.11.4 + +:warning: Support until streamlit version `1.29.0` +:point_right: For higher version, try streamlit app with embeddable python #TODO add link + +To create an executable for Streamlit app on Windows, we'll use an pyinstaller. +Here's a step-by-step guide: + +### virtual environment + +``` +# create an environment +python -m venv + +# activate an environment +.\myenv\Scripts\Activate.bat + +# install require packages +pip install -r requirements.txt + +#install pyinstaller +pip install pyinstaller +``` + +### streamlit files + +create a run_app.py and add this lines of codes +``` +from streamlit.web import cli + +if __name__=='__main__': + cli._main_run_clExplicit( + file="app.py", command_line="streamlit run" + ) + # we will create this function inside our streamlit framework + +``` + +### write function in cli.py + +Now, navigate to the inside streamlit environment + +here you go + +``` +\Lib\site-packages\streamlit\web\cli.py +``` +for using our virtual environment, add this magic function to cli.py file: +``` +#can be modify name as given in run_app.py +#use underscore at beginning +def _main_run_clExplicit(file, command_line, args=[], flag_options=[]): + main._is_running_with_streamlit = True + bootstrap.run(file, command_line, args, flag_options) +``` + +### Hook folder +Now, need to hook to get streamlit metadata +organized as folder, where the pycache infos will save +like: \hooks\hook-streamlit.py + +``` +from PyInstaller.utils.hooks import copy_metadata +datas = [] +datas += copy_metadata('streamlit') +datas += copy_metadata('pyopenms') +# can add new package e-g +datas += copy_metadata('captcha') +``` + +### compile the app +Now, ready for compilation +``` +pyinstaller --onefile --additional-hooks-dir ./hooks run_app.py --clean + +#--onefile create join binary file ?? +#will create run_app.spec file +#--clean delete cache and removed temporary files before building +#--additional-hooks-dir path to search for hook +``` + +### streamlit config +To access streamlit config create file in root +(or just can be in output folder) +.streamlit\config.toml + +``` +# content of .streamlit\config.toml +[global] +developmentMode = false + +[server] +port = 8502 +``` + +### copy necessary files to dist folder +``` +cp -r .streamlit dist/.streamlit +cp -r pages dist/pages +cp -r src dist/src +cp -r assets dist/assets +cp app.py dist/ +cp presets.json dist/ +``` + +Remove the server address from the bundled config so Streamlit uses `localhost` (default) instead of `0.0.0.0`, which doesn't work as a connect address on Windows: + +```powershell +(Get-Content dist/.streamlit/config.toml) -notmatch '^address' | Set-Content dist/.streamlit/config.toml +``` + + +### add datas in run_app.spec (.spec file) +Add DATAS to the run_app.spec just created by compilation + +``` +datas=[ + ("myenv/Lib/site-packages/altair/vegalite/v4/schema/vega-lite-schema.json","./altair/vegalite/v4/schema/"), + ("myenv/Lib/site-packages/streamlit/static", "./streamlit/static"), + ("myenv/Lib/site-packages/streamlit/runtime", "./streamlit/runtime"), + ("myenv/Lib/site-packages/pyopenms", "./pyopenms/"), + # Add new datas e-g we add in hook captcha + ("myenv/Lib/site-packages/captcha", "./captcha/") + ] +``` +### run final step to make executable +All the modifications in datas should be loaded with +``` +pyinstaller run_app.spec --clean +``` +#### 🚀 After successfully completing all these steps, the Windows executable will be available in the dist folder. + +:pencil: you can still change the configuration of streamlit app with .streamlit/config.toml file e-g provide different port, change upload size etc + +ℹ️ if problem with altair, Try version altair==4.0.1, and again compile + +## Build executable in github action automatically +Automate the process of building executables for your project with the GitHub action example [Test streamlit executable for Windows with pyinstaller](https://github.com/OpenMS/streamlit-template/blob/main/.github/workflows/test-win-exe-w-pyinstaller.yaml) diff --git a/gdpr_consent/dist/bundle.js b/gdpr_consent/dist/bundle.js index 8614457..0a48bfd 100644 --- a/gdpr_consent/dist/bundle.js +++ b/gdpr_consent/dist/bundle.js @@ -235,7 +235,7 @@ eval("__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpac /***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => { "use strict"; -eval("__webpack_require__.r(__webpack_exports__);\n/* harmony import */ var streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! streamlit-component-lib */ \"./node_modules/streamlit-component-lib/dist/index.js\");\nvar __awaiter = (undefined && undefined.__awaiter) || function (thisArg, _arguments, P, generator) {\n function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }\n return new (P || (P = Promise))(function (resolve, reject) {\n function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }\n function rejected(value) { try { step(generator[\"throw\"](value)); } catch (e) { reject(e); } }\n function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }\n step((generator = generator.apply(thisArg, _arguments || [])).next());\n });\n};\nvar __generator = (undefined && undefined.__generator) || function (thisArg, body) {\n var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;\n return g = { next: verb(0), \"throw\": verb(1), \"return\": verb(2) }, typeof Symbol === \"function\" && (g[Symbol.iterator] = function() { return this; }), g;\n function verb(n) { return function (v) { return step([n, v]); }; }\n function step(op) {\n if (f) throw new TypeError(\"Generator is already executing.\");\n while (g && (g = 0, op[0] && (_ = 0)), _) try {\n if (f = 1, y && (t = op[0] & 2 ? y[\"return\"] : op[0] ? y[\"throw\"] || ((t = y[\"return\"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;\n if (y = 0, t) op = [op[0] & 2, t.value];\n switch (op[0]) {\n case 0: case 1: t = op; break;\n case 4: _.label++; return { value: op[1], done: false };\n case 5: _.label++; y = op[1]; op = [0]; continue;\n case 7: op = _.ops.pop(); _.trys.pop(); continue;\n default:\n if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }\n if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }\n if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }\n if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }\n if (t[2]) _.ops.pop();\n _.trys.pop(); continue;\n }\n op = body.call(thisArg, _);\n } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }\n if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };\n }\n};\n\n// Defines the configuration for Klaro\nvar klaroConfig = {\n mustConsent: true,\n acceptAll: true,\n services: []\n};\n// This will make klaroConfig globally accessible\nwindow.klaroConfig = klaroConfig;\n// Function to safely access the Klaro manager\nfunction getKlaroManager() {\n var _a;\n return ((_a = window.klaro) === null || _a === void 0 ? void 0 : _a.getManager) ? window.klaro.getManager() : null;\n}\n// Waits until Klaro Manager is available\nfunction waitForKlaroManager() {\n return __awaiter(this, arguments, void 0, function (maxWaitTime, interval) {\n var startTime, klaroManager;\n if (maxWaitTime === void 0) { maxWaitTime = 5000; }\n if (interval === void 0) { interval = 100; }\n return __generator(this, function (_a) {\n switch (_a.label) {\n case 0:\n startTime = Date.now();\n _a.label = 1;\n case 1:\n if (!(Date.now() - startTime < maxWaitTime)) return [3 /*break*/, 3];\n klaroManager = getKlaroManager();\n if (klaroManager) {\n return [2 /*return*/, klaroManager];\n }\n return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, interval); })];\n case 2:\n _a.sent();\n return [3 /*break*/, 1];\n case 3: throw new Error(\"Klaro manager did not become available within the allowed time.\");\n }\n });\n });\n}\n// Helper function to handle unknown errors\nfunction handleError(error) {\n if (error instanceof Error) {\n console.error(\"Error:\", error.message);\n }\n else {\n console.error(\"Unknown error:\", error);\n }\n}\n// Tracking was accepted\nfunction callback() {\n return __awaiter(this, void 0, void 0, function () {\n var manager, return_vals, _i, _a, service, error_1;\n return __generator(this, function (_b) {\n switch (_b.label) {\n case 0:\n _b.trys.push([0, 2, , 3]);\n return [4 /*yield*/, waitForKlaroManager()];\n case 1:\n manager = _b.sent();\n if (manager.confirmed) {\n return_vals = {};\n for (_i = 0, _a = klaroConfig.services; _i < _a.length; _i++) {\n service = _a[_i];\n return_vals[service.name] = manager.getConsent(service.name);\n }\n streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentValue(return_vals);\n }\n return [3 /*break*/, 3];\n case 2:\n error_1 = _b.sent();\n handleError(error_1);\n return [3 /*break*/, 3];\n case 3: return [2 /*return*/];\n }\n });\n });\n}\n// Stores if the component has been rendered before\nvar rendered = false;\nfunction onRender(event) {\n // Klaro does not work if embedded multiple times\n if (rendered) {\n return;\n }\n rendered = true;\n var data = event.detail;\n if (data.args['google_analytics']) {\n klaroConfig.services.push({\n name: 'google-analytics',\n cookies: [\n /^_ga(_.*)?/ // we delete the Google Analytics cookies if the user declines its use\n ],\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['piwik_pro']) {\n klaroConfig.services.push({\n name: 'piwik-pro',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['matomo']) {\n klaroConfig.services.push({\n name: 'matomo',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n // Create a new script element\n var script = document.createElement('script');\n // Set the necessary attributes\n script.defer = true;\n script.type = 'application/javascript';\n script.src = 'https://cdn.kiprotect.com/klaro/v0.7/klaro.js';\n // Set the klaro config\n script.setAttribute('data-config', 'klaroConfig');\n // Append the script to the head or body\n document.head.appendChild(script);\n}\n// Attach our `onRender` handler to Streamlit's render event.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.events.addEventListener(streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.RENDER_EVENT, onRender);\n// Tell Streamlit we're ready to start receiving data. We won't get our\n// first RENDER_EVENT until we call this function.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentReady();\n// Finally, tell Streamlit to update the initial height.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setFrameHeight(1000);\n\n\n//# sourceURL=webpack://gdpr_consent/./src/main.ts?"); +eval("__webpack_require__.r(__webpack_exports__);\n/* harmony import */ var streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! streamlit-component-lib */ \"./node_modules/streamlit-component-lib/dist/index.js\");\nvar __awaiter = (undefined && undefined.__awaiter) || function (thisArg, _arguments, P, generator) {\n function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }\n return new (P || (P = Promise))(function (resolve, reject) {\n function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }\n function rejected(value) { try { step(generator[\"throw\"](value)); } catch (e) { reject(e); } }\n function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }\n step((generator = generator.apply(thisArg, _arguments || [])).next());\n });\n};\nvar __generator = (undefined && undefined.__generator) || function (thisArg, body) {\n var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;\n return g = { next: verb(0), \"throw\": verb(1), \"return\": verb(2) }, typeof Symbol === \"function\" && (g[Symbol.iterator] = function() { return this; }), g;\n function verb(n) { return function (v) { return step([n, v]); }; }\n function step(op) {\n if (f) throw new TypeError(\"Generator is already executing.\");\n while (g && (g = 0, op[0] && (_ = 0)), _) try {\n if (f = 1, y && (t = op[0] & 2 ? y[\"return\"] : op[0] ? y[\"throw\"] || ((t = y[\"return\"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;\n if (y = 0, t) op = [op[0] & 2, t.value];\n switch (op[0]) {\n case 0: case 1: t = op; break;\n case 4: _.label++; return { value: op[1], done: false };\n case 5: _.label++; y = op[1]; op = [0]; continue;\n case 7: op = _.ops.pop(); _.trys.pop(); continue;\n default:\n if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }\n if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }\n if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }\n if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }\n if (t[2]) _.ops.pop();\n _.trys.pop(); continue;\n }\n op = body.call(thisArg, _);\n } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }\n if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };\n }\n};\n\n// Defines the configuration for Klaro\nvar klaroConfig = {\n mustConsent: true,\n acceptAll: true,\n services: []\n};\n// This will make klaroConfig globally accessible\nwindow.klaroConfig = klaroConfig;\n// Function to safely access the Klaro manager\nfunction getKlaroManager() {\n var _a;\n return ((_a = window.klaro) === null || _a === void 0 ? void 0 : _a.getManager) ? window.klaro.getManager() : null;\n}\n// Waits until Klaro Manager is available\nfunction waitForKlaroManager() {\n return __awaiter(this, arguments, void 0, function (maxWaitTime, interval) {\n var startTime, klaroManager;\n if (maxWaitTime === void 0) { maxWaitTime = 5000; }\n if (interval === void 0) { interval = 100; }\n return __generator(this, function (_a) {\n switch (_a.label) {\n case 0:\n startTime = Date.now();\n _a.label = 1;\n case 1:\n if (!(Date.now() - startTime < maxWaitTime)) return [3 /*break*/, 3];\n klaroManager = getKlaroManager();\n if (klaroManager) {\n return [2 /*return*/, klaroManager];\n }\n return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, interval); })];\n case 2:\n _a.sent();\n return [3 /*break*/, 1];\n case 3: throw new Error(\"Klaro manager did not become available within the allowed time.\");\n }\n });\n });\n}\n// Helper function to handle unknown errors\nfunction handleError(error) {\n if (error instanceof Error) {\n console.error(\"Error:\", error.message);\n }\n else {\n console.error(\"Unknown error:\", error);\n }\n}\n// Tracking was accepted\nfunction callback() {\n return __awaiter(this, void 0, void 0, function () {\n var manager, return_vals, _i, _a, service, error_1;\n return __generator(this, function (_b) {\n switch (_b.label) {\n case 0:\n _b.trys.push([0, 2, , 3]);\n return [4 /*yield*/, waitForKlaroManager()];\n case 1:\n manager = _b.sent();\n if (manager.confirmed) {\n return_vals = {};\n for (_i = 0, _a = klaroConfig.services; _i < _a.length; _i++) {\n service = _a[_i];\n return_vals[service.name] = manager.getConsent(service.name);\n }\n streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentValue(return_vals);\n }\n return [3 /*break*/, 3];\n case 2:\n error_1 = _b.sent();\n handleError(error_1);\n return [3 /*break*/, 3];\n case 3: return [2 /*return*/];\n }\n });\n });\n}\n// Stores if the component has been rendered before\nvar rendered = false;\nfunction onRender(event) {\n // Klaro does not work if embedded multiple times\n if (rendered) {\n return;\n }\n rendered = true;\n var data = event.detail;\n if (data.args['google_analytics']) {\n klaroConfig.services.push({\n name: 'google-analytics',\n cookies: [\n /^_ga(_.*)?/ // we delete the Google Analytics cookies if the user declines its use\n ],\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['piwik_pro']) {\n klaroConfig.services.push({\n name: 'piwik-pro',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['matomo']) {\n klaroConfig.services.push({\n name: 'matomo',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n // Link the consent banner to the privacy policy. Setting privacyPolicyUrl\n // on the 'zz' fallback language makes Klaro render its default\n // \"To learn more, please read our privacy policy.\" text with the URL,\n // regardless of the browser locale.\n if (data.args['privacy_policy']) {\n klaroConfig.translations = {\n zz: {\n privacyPolicyUrl: data.args['privacy_policy']\n }\n };\n }\n // Create a new script element\n var script = document.createElement('script');\n // Set the necessary attributes\n script.defer = true;\n script.type = 'application/javascript';\n script.src = 'https://cdn.kiprotect.com/klaro/v0.7/klaro.js';\n // Set the klaro config\n script.setAttribute('data-config', 'klaroConfig');\n // Append the script to the head or body\n document.head.appendChild(script);\n}\n// Attach our `onRender` handler to Streamlit's render event.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.events.addEventListener(streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.RENDER_EVENT, onRender);\n// Tell Streamlit we're ready to start receiving data. We won't get our\n// first RENDER_EVENT until we call this function.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentReady();\n// Finally, tell Streamlit to update the initial height.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setFrameHeight(1000);\n\n\n//# sourceURL=webpack://gdpr_consent/./src/main.ts?"); /***/ }), diff --git a/gdpr_consent/src/main.ts b/gdpr_consent/src/main.ts index 059fef8..408f4a4 100644 --- a/gdpr_consent/src/main.ts +++ b/gdpr_consent/src/main.ts @@ -14,6 +14,7 @@ let klaroConfig: { mustConsent: boolean; acceptAll: boolean; services: Service[]; + translations?: Record; } = { mustConsent: true, acceptAll: true, @@ -125,6 +126,18 @@ function onRender(event: Event): void { ) } + // Link the consent banner to the privacy policy. Setting privacyPolicyUrl + // on the 'zz' fallback language makes Klaro render its default + // "To learn more, please read our privacy policy." text with the URL, + // regardless of the browser locale. + if (data.args['privacy_policy']) { + klaroConfig.translations = { + zz: { + privacyPolicyUrl: data.args['privacy_policy'] + } + } + } + // Create a new script element var script = document.createElement('script') diff --git a/requirements.txt b/requirements.txt index aac2879..7caa20a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,9 +16,9 @@ cachetools==5.5.2 # via streamlit captcha==0.7.1 # via src (pyproject.toml) -certifi==2025.1.31 +certifi==2025.8.3 # via requests -charset-normalizer==3.4.1 +charset-normalizer==3.4.3 # via requests click==8.1.8 # via streamlit @@ -26,11 +26,11 @@ contourpy==1.3.1 # via matplotlib cycler==0.12.1 # via matplotlib -fonttools==4.56.0 +fonttools==4.59.2 # via matplotlib gitdb==4.0.12 # via gitpython -gitpython==3.1.44 +gitpython==3.1.45 # via streamlit idna==3.10 # via requests @@ -48,7 +48,7 @@ markupsafe==3.0.2 # via jinja2 matplotlib==3.10.1 # via pyopenms -narwhals==1.32.0 +narwhals==2.5.0 # via altair numpy==1.26.4 # via @@ -59,7 +59,7 @@ numpy==1.26.4 # pyopenms # src (pyproject.toml) # streamlit -packaging==24.2 +packaging==25.0 # via # altair # matplotlib @@ -77,7 +77,7 @@ pillow==11.1.0 # streamlit plotly==5.22.0 # via src (pyproject.toml) -protobuf==5.29.4 +protobuf==6.32.0 # via streamlit psutil==7.0.0 # via src (pyproject.toml) @@ -85,7 +85,7 @@ pyarrow==19.0.1 # via streamlit pydeck==0.9.1 # via streamlit -pyopenms>=3.0.0,<3.5 +pyopenms==3.5.0 # via src (pyproject.toml) pyopenms-viz==1.0.0 # via src (pyproject.toml) @@ -111,7 +111,7 @@ six==1.17.0 # via python-dateutil smmap==5.0.2 # via gitdb -streamlit==1.43.0 +streamlit==1.49.1 # via # src (pyproject.toml) # streamlit-js-eval @@ -140,13 +140,19 @@ xlsxwriter streamlit_plotly_events scipy scikit-learn -openms-insight>=0.1.13 +openms-insight==0.2.0 polars>=1.0.0 +# Forces polars to prefer its most CPU-compatible native runtime. +# Without this, polars can select an AVX-optimized runtime (e.g. runtime-32) +# that access-violation crashes (0xC0000005 in _polars_runtime.pyd) on some +# CPUs (seen on AMD Threadripper PRO 3995WX on Windows) during dataframe ops. +polars-runtime-compat cython easypqp>=0.1.34 pyprophet>=2.2.0 mygene +statsmodels + # Redis Queue dependencies (for online mode) redis>=5.0.0 rq>=1.16.0 -statsmodels \ No newline at end of file diff --git a/settings.json b/settings.json index f792ddc..60424cb 100644 --- a/settings.json +++ b/settings.json @@ -1,8 +1,13 @@ { - "app-name": "quantms-web (DDA-LFQ)", + "app-name": "OpenMS WebApp Template", "github-user": "OpenMS", - "version": "1.0", + "version": "1.1.1", "repository-name": "streamlit-template", + "legal_links": { + "impressum": "https://openms.de/impressum", + "privacy": "https://openms.de/privacy", + "terms": "https://openms.de/terms" + }, "analytics": { "google-analytics": { "enabled": false, @@ -22,6 +27,7 @@ "enable_workspaces": true, "test": true, "workspaces_dir": "..", + "local_data_dir": "", "queue_settings": { "default_timeout": 7200, "result_ttl": 86400 diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index ebf7e65..120efb0 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -1,5 +1,6 @@ import streamlit as st from pathlib import Path +import re import pandas as pd import plotly.express as px from streamlit_plotly_events import plotly_events @@ -409,6 +410,7 @@ def render_tmt_tabs(self): "quantification:isotope_correction": "false", }, tool_instance_name="IsobaricAnalyzer-TMT", + reactive=True, ) with t[1]: comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", @@ -559,64 +561,62 @@ def render_tmt_tabs(self): ) with t[10]: st.markdown("### 🧪 TMT Sample Group Assignment") - - # 1. Determine TMT type (e.g., tmt10plex, tmt16plex) - target_key = f"{self.parameter_manager.topp_param_prefix}IsobaricAnalyzer:1:type" - selected_tmt = st.session_state.get(target_key, "tmt12plex") - - if "tmt" in selected_tmt: - import re - # Extract the number to determine the plex count - num_plex_match = re.search(r'\d+', selected_tmt) - if num_plex_match: - num_plex = int(num_plex_match.group()) - all_channels = [f"sample{i+1}" for i in range(num_plex)] - - st.info( - "Enter a group name for each TMT channel.\n\n" - "Type **'skip'** for channels you wish to skip. (e.g., control, case, skip)" - ) - # 2. Create an input_widget for each channel (automatically saved to params.json) - cols = st.columns(2) - for i, ch in enumerate(all_channels): - with cols[i % 2]: + latest_params = self.parameter_manager.get_parameters_from_json() + type_key = ( + f"{self.parameter_manager.topp_param_prefix}" + "IsobaricAnalyzer-TMT:1:type" + ) + selected_type = str( + st.session_state.get(type_key) + or latest_params.get("IsobaricAnalyzer-TMT", {}).get("type") + or "tmt11plex" + ).lower() + + m = re.search(r'\d+', selected_type) + is_supported_type = any(label in selected_type for label in ["tmt", "itraq"]) + if not m or not is_supported_type: + st.warning("Please select a supported isobaric type in the IsobaricAnalyzer tab first.") + else: + num_plex = int(m.group()) + channels = [f"sample{i+1}" for i in range(num_plex)] + st.caption(f"Isobaric type: **{selected_type}** - {num_plex} channels") + st.info("Assign a group name to each channel. Use **'skip'** to exclude a channel.") + + for row_start in range(0, num_plex, 2): + c1, c2 = st.columns(2) + + left_idx = row_start + left_channel = channels[left_idx] + with c1: + self.ui.input_widget( + key=f"TMT-group-{left_channel}", + default="", + name=f"Group for channel {left_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + right_idx = row_start + 1 + if right_idx < num_plex: + right_channel = channels[right_idx] + with c2: self.ui.input_widget( - key=f"TMT-group-{ch}", + key=f"TMT-group-{right_channel}", default="", - name=f"Group for {ch}", + name=f"Group for channel {right_idx + 1}", widget_type="text", - help="Enter group name or 'skip' to ignore this channel.", + help="e.g. control, case, skip", ) - # 3. Read values from params.json and construct a dictionary in tmt_group_map format - # (This can be used later to filter DataFrames in subsequent logic) - self.params = self.parameter_manager.get_parameters_from_json() - - tmt_group_map = {} - for i, ch in enumerate(all_channels): - # Retrieve stored value (default is empty string) - group_val = self.params.get(f"TMT-group-{ch}", "") - tmt_group_map[str(i)] = group_val - - # For data inspection (remove if not needed) - if st.checkbox("Show current TMT mapping"): - st.json(tmt_group_map) - - # 4. Clean up parameters from unused/previous TMT settings - all_possible_channel_keys = {f"TMT-group-{ch}" for ch in all_channels} - orphaned_keys = [ - k for k in self.params.keys() - if k.startswith("TMT-group-") and k not in all_possible_channel_keys - ] - - if orphaned_keys: - for key in orphaned_keys: - del self.params[key] - self.parameter_manager.save_parameters() - - else: - st.warning("Please select a TMT type in the parameters first.") + # Remove orphaned params from a previously selected larger plex + self.params = self.parameter_manager.get_parameters_from_json() + valid_keys = {f"TMT-group-{ch}" for ch in channels} + orphaned = [k for k in self.params if k.startswith("TMT-group-") and k not in valid_keys] + if orphaned: + for k in orphaned: + del self.params[k] + self.parameter_manager.save_parameters() def execution(self) -> bool: """ diff --git a/src/common/admin.py b/src/common/admin.py index 2408c0a..a4414b9 100644 --- a/src/common/admin.py +++ b/src/common/admin.py @@ -9,6 +9,7 @@ from pathlib import Path import streamlit as st +from streamlit.errors import StreamlitSecretNotFoundError def is_admin_configured() -> bool: @@ -20,7 +21,7 @@ def is_admin_configured() -> bool: """ try: return bool(st.secrets.get("admin", {}).get("password")) - except (FileNotFoundError, KeyError): + except (FileNotFoundError, KeyError, StreamlitSecretNotFoundError): return False except Exception: return False diff --git a/src/common/captcha_.py b/src/common/captcha_.py index 498b133..282e124 100644 --- a/src/common/captcha_.py +++ b/src/common/captcha_.py @@ -186,7 +186,7 @@ def add_page(main_script_path_str: str, page_name: str) -> None: # define the function for the captcha control -def captcha_control(): +def captcha_control(privacy_policy_url: str = ""): """ Control and verification of a CAPTCHA to ensure the user is not a robot. @@ -199,6 +199,10 @@ def captcha_control(): The CAPTCHA text is generated as a session state and should not change during refreshes. + Args: + privacy_policy_url (str, optional): URL shown as the privacy policy link + in the GDPR consent banner. Defaults to "". + Returns: None """ @@ -214,7 +218,10 @@ def captcha_control(): with st.spinner(): # Ask for consent st.session_state.tracking_consent = consent_component( - google_analytics=ga, piwik_pro=pp, matomo=mt + google_analytics=ga, + piwik_pro=pp, + matomo=mt, + privacy_policy=privacy_policy_url, ) if st.session_state.tracking_consent is None: # No response by user yet diff --git a/src/common/common.py b/src/common/common.py index 643a224..c048064 100644 --- a/src/common/common.py +++ b/src/common/common.py @@ -31,6 +31,37 @@ # Detect system platform OS_PLATFORM = sys.platform +# Default legal/GDPR page links. These point to the centrally maintained +# official OpenMS pages. Forks that self-host should override them via the +# "legal_links" key in settings.json (an Impressum must name the actual +# operator). The defaults live here too — not only in settings.json — so that +# downstream apps built from an older settings.json without a "legal_links" +# key still inherit working legal links by default. +DEFAULT_LEGAL_LINKS = { + "impressum": "https://openms.de/impressum", + "privacy": "https://openms.de/privacy", + "terms": "https://openms.de/terms", +} + + +def get_legal_links() -> dict[str, str]: + """ + Return the legal page URLs (Impressum, Privacy Policy, Terms of Use). + + Values from the "legal_links" object in settings.json override the + built-in OpenMS defaults. Empty override values are ignored so a blank + entry can't erase a default. + + Returns: + dict[str, str]: Mapping of "impressum", "privacy" and "terms" to URLs. + """ + overrides = ( + st.session_state.settings.get("legal_links", {}) + if "settings" in st.session_state + else {} + ) + return {**DEFAULT_LEGAL_LINKS, **{k: v for k, v in overrides.items() if v}} + def is_safe_workspace_name(name: str) -> bool: """ @@ -519,7 +550,7 @@ def page_setup(page: str = "") -> dict[str, Any]: # Render the sidebar params = render_sidebar(page) - captcha_control() + captcha_control(privacy_policy_url=get_legal_links()["privacy"]) # If run in hosted mode, show captcha as long as it has not been solved # if not "local" in sys.argv: @@ -532,7 +563,7 @@ def page_setup(page: str = "") -> dict[str, Any]: "controllo" in params.keys() and params["controllo"] == False ): # Apply captcha by calling the captcha_control function - captcha_control() + captcha_control(privacy_policy_url=get_legal_links()["privacy"]) return params @@ -764,6 +795,19 @@ def change_workspace(): f'
{app_name}
Version: {version_info}
', unsafe_allow_html=True, ) + + # Legal links (Impressum, Privacy Policy, Terms of Use), shown on every + # page. URLs are configurable via "legal_links" in settings.json. + links = get_legal_links() + st.markdown( + '
' + f'Impressum · ' + f'Privacy Policy · ' + f'Terms of Use' + "
", + unsafe_allow_html=True, + ) return params diff --git a/src/common/results_helpers.py b/src/common/results_helpers.py index 02d7bd4..2d38ad9 100644 --- a/src/common/results_helpers.py +++ b/src/common/results_helpers.py @@ -5,11 +5,8 @@ import numpy as np import streamlit as st from pathlib import Path -from scipy.stats import ttest_ind from pyopenms import IdXMLFile, MSExperiment, MzMLFile from src.workflow.ParameterManager import ParameterManager -from statsmodels.stats.multitest import multipletests -from statsmodels.stats.multitest import multipletests def get_workflow_dir(workspace): """Get the workflow directory path.""" @@ -185,12 +182,15 @@ def build_spectra_cache(mzml_dir: Path, filename_to_index: dict) -> tuple[pl.Dat @st.cache_data -def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: - """Load CSV, compute stats (log2FC, p-value), build pivot_df and expr_df. +def load_abundance_data(workspace_path: str, csv_mtime: float, params_mtime: float = 0.0) -> tuple | None: + """Load CSV and build abundance matrices for downstream preprocessing. Args: workspace_path: Path to the workspace directory csv_mtime: Modification time of CSV file (used as cache key) + params_mtime: Modification time of params.json (used as cache key so + changing group assignments in Configure invalidates the cache + even when the CSV itself hasn't changed) Returns: Tuple of (pivot_df, expr_df, group_map) or None if data unavailable @@ -200,7 +200,7 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - workflow_params = parameter_manager.get_parameters_from_json() + workflow_params = parameter_manager.get_parameters_from_json() analysis_mode = workflow_params.get("analysis-mode", "LFQ") if analysis_mode == "LFQ": @@ -221,7 +221,9 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: if df.empty: return None - # Get group mapping from parameters + # Get optional group mapping from parameters. + # Group information is not required at this stage; statistical testing + # happens in the Statistical page. param_manager = ParameterManager(workflow_dir) params = param_manager.get_parameters_from_json() group_map = { @@ -230,57 +232,23 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: if key.startswith("mzML-group-") and value } - if not group_map: - return None - df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) - df["Group"] = df["Reference"].map(group_map) - df = df.dropna(subset=["Group"]) - - groups = sorted(df["Group"].unique()) - - if len(groups) < 2: - return None - - group1, group2 = groups[:2] - - # Compute statistics per protein - stats_rows = [] - for protein, protein_df in df.groupby("ProteinName"): - g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values - g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values - - if len(g1_vals) < 2 or len(g2_vals) < 2: - pval = np.nan - else: - _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False) - - mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan - mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan - - log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan - - stats_rows.append({ - "ProteinName": protein, - "log2FC": log2fc, - "p-value": pval, - }) - - stats_df = pd.DataFrame(stats_rows) - if not stats_df.empty: - mask = stats_df["p-value"].notna() - if mask.any(): - _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh") - stats_df.loc[mask, "p-adj"] = p_adj - else: - stats_df["p-adj"] = np.nan - - # Order samples by group (group2 first, then group1) - sample_group_df = df[["Sample", "Group"]].drop_duplicates() - group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist() - group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist() - all_samples = group2_samples + group1_samples + # Build sample display order. + if group_map: + sample_group_df = df[["Sample", "Reference"]].drop_duplicates() + sample_group_df["Group"] = sample_group_df["Reference"].map(group_map) + grouped_samples = [] + for grp in sorted(sample_group_df["Group"].dropna().unique()): + grouped_samples.extend( + sample_group_df[sample_group_df["Group"] == grp]["Sample"].tolist() + ) + remaining_samples = [ + s for s in sorted(df["Sample"].unique()) if s not in grouped_samples + ] + all_samples = grouped_samples + remaining_samples + else: + all_samples = sorted(df["Sample"].unique()) # Build pivot table pivot_list = [] @@ -299,8 +267,7 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: pivot_list.append(row) pivot_df = pd.DataFrame(pivot_list) - pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left") - pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]] + pivot_df = pivot_df[["ProteinName"] + all_samples + ["PeptideSequence"]] # Build expression matrix (log2-transformed) expr_df = pivot_df.set_index("ProteinName")[all_samples] @@ -309,7 +276,7 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: expr_df = expr_df.dropna() return pivot_df, expr_df, group_map - + else: if not quant_dir.exists(): return None @@ -330,7 +297,7 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: # ratio column removal df = df.loc[:, ~df.columns.str.contains('ratio', case=False)] - + # exclude_indices = st.session_state.get("tmt_exclude_indices", []) # group_map = st.session_state.get("tmt_group_map", {}) # Get group mapping from parameters @@ -362,10 +329,6 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: # st.write("exclude_indices:", exclude_indices) # st.write("group_map:", group_map) - if not group_map: - st.warning("⚠️ Group mapping information is missing. Please configure sample groups in the Setup page.") - return None - if exclude_indices: # st.write("Current columns:", df.columns.tolist()) # st.write("Number of columns:", len(df.columns)) @@ -376,115 +339,25 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: else: df_cleaned = df.copy() - if group_map: - # Create new row data (defaulting to empty strings) - # Create a list with the same length as the column order of df_cleaned - new_row = [""] * len(df_cleaned.columns) - new_row[0] = "Group" - - # Get the column names of the current dataframe as a list - current_cols = df_cleaned.columns.tolist() - original_cols = df.columns.tolist() - - for col_name in current_cols[start_column_offset:]: - # Check the original index position of this column - original_idx = original_cols.index(col_name) - start_column_offset - col_pos = current_cols.index(col_name) - new_row[col_pos] = group_map.get(original_idx, "NA") - - # Insert the row at the top of the dataframe - # Create a new DF and concatenate to prepend the row to existing data - group_df = pd.DataFrame([new_row], columns=df_cleaned.columns) - df_with_groups = pd.concat([group_df, df_cleaned], ignore_index=True) - - # drop_msg = f"{len(exclude_indices)} channels dropped" if exclude_indices else "No channels dropped" - # st.success(f"✅ {drop_msg} and Group names have been inserted at the top of the data.") - - # st.write("### Data Preview with Group Information") - # st.dataframe(df_with_groups.head(10)) - - if group_map and len(set(group_map.values())) >= 2: - # Prepare data for calculation - # Extract group information from row 0 of df_with_groups (the newly added Group row) - # Actual sample data starts from the 5th column (index 4) - group_info_row = df_with_groups.iloc[0] - - # Get unique group names (excluding NA) - unique_groups = sorted([g for g in set(group_map.values()) if g != "NA"]) - g1_name, g2_name = unique_groups[0], unique_groups[1] - - # Extract numerical data for statistical calculation (from row 1 and column index 4 onwards) - # Convert to numeric type (to prevent calculation errors) - numeric_data = df_with_groups.iloc[1:, 4:].apply(pd.to_numeric, errors='coerce') - - # Column indexing by group - # Categorize columns based on the values in the Group row - g1_cols = [col for col in numeric_data.columns if group_info_row[col] == g1_name] - g2_cols = [col for col in numeric_data.columns if group_info_row[col] == g2_name] - - # Calculate log2FC and p-value for each row - def run_stats(row): - v1 = row[g1_cols].dropna() - v2 = row[g2_cols].dropna() - - # log2FC (Group2 / Group1) - m1, m2 = v1.mean(), v2.mean() - l2fc = np.log2(m2 / m1) if m1 > 0 and m2 > 0 else np.nan - - # p-value (T-test) - if len(v1) > 1 and len(v2) > 1: - _, pval = ttest_ind(v1, v2, equal_var=False) - else: - pval = np.nan - return pd.Series([l2fc, pval]) - - stats_results = numeric_data.apply(run_stats, axis=1) - stats_results.columns = ['log2FC', 'p-value'] - # Add Adjusted p-value (FDR) calculation - if not stats_results['p-value'].isna().all(): - # Select only rows that contain p-values - mask = stats_results['p-value'].notna() - # Apply Benjamini-Hochberg (BH) correction - _, p_adj, _, _ = multipletests(stats_results.loc[mask, 'p-value'], method='fdr_bh') - stats_results.loc[mask, 'p-adj'] = p_adj - else: - stats_results['p-adj'] = np.nan - - # Construct the final dataframe (Based on df_cleaned - excluding the group row) - # Insert calculation results into the 2nd and 3rd column positions - pivot_df = df_cleaned.copy() - pivot_df.insert(1, "log2FC", stats_results['log2FC'].values) - pivot_df.insert(2, "p-value", stats_results['p-value'].values) - pivot_df.insert(3, "p-adj", stats_results['p-adj'].values) - - # st.success(f"Analysis Complete: {g1_name} (n={len(g1_cols)}) vs {g2_name} (n={len(g2_cols)})") - - # Set the first column ('protein') of final_df as the index - protein_col = pivot_df.columns[0] - sample_cols = current_cols[start_column_offset:] # Identify actual sample column names - - # Select sample columns and create a matrix - expr_df = pivot_df.set_index(protein_col)[sample_cols] - - # Replace 0 with NaN (to prevent log transformation errors) - expr_df = expr_df.replace(0, np.nan) - - # Log2 transformation (data normalization) - expr_df = np.log2(expr_df + 1) - - # Remove proteins (rows) with any missing values - expr_df = expr_df.dropna() - - return pivot_df, expr_df, group_map - else: - st.warning("⚠️ At least two distinct groups are required for statistical analysis.") - else: - st.warning("⚠️ No group mapping information is set. Please check the Configure page.") - return None + current_cols = df_cleaned.columns.tolist() + sample_cols = current_cols[start_column_offset:] + + # Ensure sample columns are numeric for downstream preprocessing/statistics. + pivot_df = df_cleaned.copy() + if sample_cols: + pivot_df[sample_cols] = pivot_df[sample_cols].apply(pd.to_numeric, errors='coerce') + + protein_col = pivot_df.columns[0] + expr_df = pivot_df.set_index(protein_col)[sample_cols] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() + + return pivot_df, expr_df, group_map def get_abundance_data(workspace: Path) -> tuple | None: - """Wrapper that handles cache key (workspace + CSV mtime). + """Wrapper that handles cache key (workspace + CSV mtime + params mtime). Args: workspace: Path to the workspace directory @@ -503,4 +376,49 @@ def get_abundance_data(workspace: Path) -> tuple | None: return None csv_mtime = csv_files[0].stat().st_mtime - return load_abundance_data(str(workspace), csv_mtime) + + params_file = workflow_dir / "params.json" + params_mtime = params_file.stat().st_mtime if params_file.exists() else 0.0 + + return load_abundance_data(str(workspace), csv_mtime, params_mtime) + + +def get_id_column(workspace: Path, pivot_df: pd.DataFrame) -> str: + """Resolve the protein/row identifier column for the active analysis mode. + + LFQ reports always use "ProteinName"; TMT reports use whatever the + report's first column is actually named (e.g. "protein"). + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + return "ProteinName" if analysis_mode == "LFQ" else pivot_df.columns[0] + + +def get_sample_group_map(workspace: Path, pivot_df: pd.DataFrame, group_map: dict) -> dict: + """Normalize group_map into {actual_sample_column_name: group_name}. + + LFQ group_map keys are already clean sample names (optionally with a + ".mzML" suffix). TMT group_map keys are 0-based channel indices that must + be matched against the report's actual "sampleN[...]" column names. + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + + if analysis_mode == "LFQ": + return { + k[:-5] if k.endswith(".mzML") else k: v + for k, v in group_map.items() + } + + actual_sample_names = pivot_df.columns.tolist() + norm_map = {} + for k, v in group_map.items(): + try: + sample_idx = int(k) + 1 + except (TypeError, ValueError): + continue + target_substring = f"sample{sample_idx}[" + real_full_name = next((name for name in actual_sample_names if target_substring in name), None) + if real_full_name: + norm_map[real_full_name] = v if v and v.strip() else "Unassigned" + return norm_map diff --git a/src/mzmlfileworkflow.py b/src/mzmlfileworkflow.py new file mode 100644 index 0000000..94faa2f --- /dev/null +++ b/src/mzmlfileworkflow.py @@ -0,0 +1,107 @@ +import streamlit as st +from pathlib import Path +import pyopenms as poms +import pandas as pd +import time +from datetime import datetime +from src.common.common import reset_directory, show_fig, show_table +import plotly.express as px + + +def mzML_file_get_num_spectra(filepath): + """ + Load an mzML file, retrieve the number of spectra, and return it. + + This function loads an mzML file specified by `filepath` and extracts the number of spectra + contained within the file using the OpenMS library. It temporarily pauses for 2 seconds to + simulate a heavy task before retrieving the number of spectra. + + Args: + filepath (str): The path to the mzML file to be loaded and analyzed. + + Returns: + int: The number of spectra present in the mzML file. + """ + exp = poms.MSExperiment() + poms.MzMLFile().load(filepath, exp) + time.sleep(2) + return exp.size() + + +def run_workflow(params, result_dir): + """Load each mzML file into pyOpenMS Experiment and get the number of spectra.""" + + result_dir = Path(result_dir, datetime.now().strftime("%Y-%m-%d %H_%M_%S")) + # delete old workflow results and set new directory + reset_directory(result_dir) + + # collect spectra numbers + num_spectra = [] + + # use st.status to print info while running the workflow + with st.status( + "Loading mzML files and getting number of spectra...", expanded=True + ) as status: + # get selected mzML files from parameters + for file in params["example-workflow-selected-mzML-files"]: + # logging file name in status + st.write(f"Reading mzML file: {file} ...") + + # reading mzML file, getting num spectra and adding some extra time + num_spectra.append( + mzML_file_get_num_spectra( + str( + Path( + st.session_state["workspace"], "mzML-files", file + ".mzML" + ) + ) + ) + ) + + # set status as complete and collapse box + status.update(label="Complete!", expanded=False) + + # create and save result dataframe + df = pd.DataFrame( + { + "filenames": params["example-workflow-selected-mzML-files"], + "number of spectra": num_spectra, + } + ) + df.to_csv(Path(result_dir, "result.tsv"), sep="\t", index=False) + +@st.fragment +def result_section(result_dir): + if not Path(result_dir).exists(): + st.error("No results to show yet. Please run a workflow first!") + return + + date_strings = [f.name for f in Path(result_dir).iterdir() if f.is_dir()] + + result_dirs = sorted(date_strings, key=lambda date: datetime.strptime(date, "%Y-%m-%d %H_%M_%S"))[::-1] + + run_dir = st.selectbox("select result from run", result_dirs) + + if run_dir is None: + st.error("Please select a result from a run!") + return + + result_dir = Path(result_dir, run_dir) + # visualize workflow results if there are any + result_file_path = Path(result_dir, "result.tsv") + + if result_file_path.exists(): + df = pd.read_csv(result_file_path, sep="\t", index_col="filenames") + + if not df.empty: + tabs = st.tabs(["📁 data", "📊 plot"]) + + with tabs[0]: + show_table(df, "mzML-workflow-result") + + with tabs[1]: + fig = px.bar(df) + st.info( + "💡 Download figure with camera icon in top right corner. File format can be specified in settings." + ) + show_fig(fig, "mzML-workflow-results") \ No newline at end of file diff --git a/src/peptide_mz_calculator.py b/src/peptide_mz_calculator.py new file mode 100644 index 0000000..5b75c57 --- /dev/null +++ b/src/peptide_mz_calculator.py @@ -0,0 +1,107 @@ +""" +Peptide M/Z Calculator Backend + +This module provides backend functions for peptide mass spectrometry calculations +using pyOpenMS AASequence.fromString() directly with minimal parsing overhead. +""" + +from typing import Dict, Any, Tuple +import pyopenms as poms + + +def calculate_peptide_mz(sequence: str, charge_state: int) -> Dict[str, Any]: + """Calculate m/z ratio for a peptide using AASequence.fromString() directly. + + Args: + sequence: Peptide sequence string (AASequence.fromString() compatible) + charge_state: Charge state for m/z calculation + + Returns: + Dictionary with calculation results + + Raises: + ValueError: If sequence is invalid or charge state is invalid + """ + sequence = sequence.strip() + if not sequence: + raise ValueError("Peptide sequence cannot be empty") + + if charge_state < 1: + raise ValueError("Charge state must be a positive integer") + + try: + # Use AASequence.fromString() directly - it supports many formats natively + aa_sequence = poms.AASequence.fromString(sequence) + except Exception as e: + raise ValueError(f"Invalid sequence format: {str(e)}") from e + + # Calculate properties + mz_ratio = aa_sequence.getMZ(charge_state) + mono_weight = aa_sequence.getMonoWeight() + formula = aa_sequence.getFormula() + + # Extract clean amino acid sequence for composition + unmodified_aa_sequence = aa_sequence.toUnmodifiedString() + + # Calculate amino acid composition + aa_composition = {} + for aa in unmodified_aa_sequence: + aa_composition[aa] = aa_composition.get(aa, 0) + 1 + + return { + "mz_ratio": mz_ratio, + "monoisotopic_mass": mono_weight, + "molecular_formula": formula.toString(), + "charge_state": charge_state, + "sequence_length": len(unmodified_aa_sequence), + "aa_composition": aa_composition, + "success": True, + } + +def calculate_peptide_mz_range( + sequence: str, + charge_range: Tuple[int, int] +) -> Dict[str, Any]: + """Calculate m/z ratios for multiple charge states. + + Args: + sequence: Peptide sequence string + charge_range: Tuple of (min_charge, max_charge) inclusive + + Returns: + Dictionary containing results for all charge states + """ + min_charge, max_charge = charge_range + charge_results = {} + + # Calculate for each charge state + for charge in range(min_charge, max_charge + 1): + result = calculate_peptide_mz(sequence, charge) + charge_results[charge] = result + + # Use first result as base and add charge_results + base_result = charge_results[min_charge] + return { + **base_result, + "charge_results": charge_results, + "charge_range": charge_range, + } + + +def validate_sequence(sequence: str) -> Tuple[bool, str]: + """Validate if sequence can be parsed by AASequence.fromString(). + + Args: + sequence: Sequence string to validate + + Returns: + Tuple of (is_valid, error_message) + """ + if not sequence.strip(): + return False, "Sequence cannot be empty" + + try: + poms.AASequence.fromString(sequence.strip()) + return True, "" + except Exception as e: + return False, f"Invalid sequence format: {str(e)}" diff --git a/src/python-tools/example.py b/src/python-tools/example.py new file mode 100644 index 0000000..50a7b47 --- /dev/null +++ b/src/python-tools/example.py @@ -0,0 +1,67 @@ +import json +import sys + +############################ +# default paramter values # +########################### +# +# Mandatory keys for each parameter +# key: a unique identifier +# value: the default value +# +# Optional keys for each parameter +# name: the name of the parameter +# hide: don't show the parameter in the parameter section (e.g. for input/output files) +# options: a list of valid options for the parameter +# min: the minimum value for the parameter (int and float) +# max: the maximum value for the parameter (int and float) +# step_size: the step size for the parameter (int and float) +# help: a description of the parameter +# widget_type: the type of widget to use for the parameter (default: auto) +# advanced: whether or not the parameter is advanced (default: False) + +DEFAULTS = [ + {"key": "in", "value": [], "help": "Input files for Python Script.", "hide": True}, + {"key": "out", "value": [], "help": "Output files for Python Script.", "hide": True}, + { + "key": "number-slider", + "name": "number of features", + "value": 6, + "min": 2, + "max": 10, + "help": "How many features to consider.", + "widget_type": "slider", + "step_size": 2, + }, + { + "key": "selectbox-example", + "name": "select something", + "value": "a", + "options": ["a", "b", "c"], + }, + { + "key": "adavanced-input", + "name": "advanced parameter", + "value": 5, + "step_size": 5, + "help": "An advanced example parameter.", + "advanced": True, + }, + { + "key": "checkbox", "value": True, "name": "boolean" + } +] + +def get_params(): + if len(sys.argv) > 1: + with open(sys.argv[1], "r") as f: + return json.load(f) + else: + return {} + +if __name__ == "__main__": + params = get_params() + # Add code here: + print("Writing stdout which will get logged...") + print("Parameters for this example Python tool:") + print(json.dumps(params, indent=4)) \ No newline at end of file diff --git a/src/python-tools/export_consensus_feature_df.py b/src/python-tools/export_consensus_feature_df.py new file mode 100644 index 0000000..9f0ceb1 --- /dev/null +++ b/src/python-tools/export_consensus_feature_df.py @@ -0,0 +1,46 @@ +import json +import sys +from pyopenms import ConsensusXMLFile, ConsensusMap +from pathlib import Path + +############################ +# default paramter values # +########################### +# +# Mandatory keys for each parameter +# key: a unique identifier +# value: the default value +# +# Optional keys for each parameter +# name: the name of the parameter +# hide: don't show the parameter in the parameter section (e.g. for input/output files) +# options: a list of valid options for the parameter +# min: the minimum value for the parameter (int and float) +# max: the maximum value for the parameter (int and float) +# step_size: the step size for the parameter (int and float) +# help: a description of the parameter +# widget_type: the type of widget to use for the parameter (default: auto) +# advanced: whether or not the parameter is advanced (default: False) + +DEFAULTS = [ + {"key": "in", "value": "", "help": "Input consensusXML file.", "hide": True}, +] + +def get_params(): + if len(sys.argv) > 1: + with open(sys.argv[1], "r") as f: + return json.load(f) + else: + return {} + +if __name__ == "__main__": + params = get_params() + # Add code here: + cm = ConsensusMap() + ConsensusXMLFile().load(params["in"], cm) + df = cm.get_df() + df = df.rename(columns={col: Path(col).name for col in df.columns}) + df = df.reset_index() + df = df.drop(columns=["id", "sequence"]) + df.insert(0, "metabolite", df.apply(lambda x: f"{round(x['mz'], 4)}@{round(x['rt'], 2)}", axis=1)) + df.to_csv(Path(params["in"]).with_suffix(".tsv"), sep="\t", index=False) \ No newline at end of file diff --git a/src/run_subprocess.py b/src/run_subprocess.py new file mode 100644 index 0000000..a5f25df --- /dev/null +++ b/src/run_subprocess.py @@ -0,0 +1,57 @@ +import streamlit as st +import subprocess + + +def run_subprocess(args: list[str], result_dict: dict) -> None: + """ + Run a subprocess and capture its output. + + Args: + args (list[str]): The command and its arguments as a list of strings. + variables (list[str]): Additional variables needed for the subprocess (not used in this code). + result_dict dict: A dictionary to store the success status (bool) and the captured log (str). + + Returns: + None + """ + + # Run the subprocess and capture its output + process = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True + ) + + # Lists to store the captured standard output and standard error + stdout_ = [] + stderr_ = [] + + # Capture the standard output of the subprocess + while True: + output = process.stdout.readline() + if output == "" and process.poll() is not None: + break + if output: + # Print every line of standard output on the Streamlit page + st.text(output.strip()) + # Append the line to store in the log + stdout_.append(output.strip()) + + # Capture the standard error of the subprocess + while True: + error = process.stderr.readline() + if error == "" and process.poll() is not None: + break + if error: + # Print every line of standard error on the Streamlit page, marking it as an error + st.error(error.strip()) + # Append the line to store in the log of errors + stderr_.append(error.strip()) + + # Check if the subprocess ran successfully (return code 0) + if process.returncode == 0: + result_dict["success"] = True + # Save all lines from standard output to the log + result_dict["log"] = " ".join(stdout_) + else: + result_dict["success"] = False + # Save all lines from standard error to the log, even if the process encountered an error + result_dict["log"] = " ".join(stderr_) diff --git a/src/simpleworkflow.py b/src/simpleworkflow.py new file mode 100644 index 0000000..0eccb9e --- /dev/null +++ b/src/simpleworkflow.py @@ -0,0 +1,13 @@ +import time + +import numpy as np +import pandas as pd +import streamlit as st + + +@st.cache_data +def generate_random_table(x, y): + """Example for a cached table""" + df = pd.DataFrame(np.random.randn(x, y)) + time.sleep(2) + return df diff --git a/src/view.py b/src/view.py new file mode 100644 index 0000000..3582b1a --- /dev/null +++ b/src/view.py @@ -0,0 +1,327 @@ +import numpy as np +import pandas as pd +from pathlib import Path +import plotly.express as px +import plotly.graph_objects as go +import streamlit as st +import pyopenms as poms +from src.common.common import show_fig, display_large_dataframe +from typing import Union + + +def get_df(file: Union[str, Path]) -> pd.DataFrame: + """ + Load a Mass Spectrometry (MS) experiment from a given mzML file and return + a pandas dataframe representation of the experiment. + + Args: + file (Union[str, Path]): The path to the mzML file to load. + + Returns: + pd.DataFrame: A pandas DataFrame with the following columns: "mslevel", + "precursormz", "mzarray", and "intarray". The "mzarray" and "intarray" + columns contain NumPy arrays with the m/z and intensity values for each + spectrum in the mzML file, respectively. + """ + exp = poms.MSExperiment() + poms.MzMLFile().load(str(file), exp) + df_spectra = exp.get_df() + df_spectra.rename(columns={ + 'rt': 'RT', + 'ms_level': 'MS level', + 'mz_array': 'mzarray', + 'intensity_array': 'intarray', + }, inplace=True) + precs = [] + for spec in exp: + p = spec.getPrecursors() + if p: + precs.append(p[0].getMZ()) + else: + precs.append(np.nan) + df_spectra["precursor m/z"] = precs + + # Drop spectra without peaks + df_spectra = df_spectra[df_spectra["mzarray"].apply(lambda x: len(x) > 0)] + + df_spectra["max intensity m/z"] = df_spectra.apply( + lambda x: x["mzarray"][x["intarray"].argmax()], axis=1 + ) + if not df_spectra.empty: + st.session_state["view_spectra"] = df_spectra + else: + st.session_state["view_spectra"] = pd.DataFrame() + exp_ms2 = poms.MSExperiment() + exp_ms1 = poms.MSExperiment() + for spec in exp: + if spec.getMSLevel() == 1: + exp_ms1.addSpectrum(spec) + elif spec.getMSLevel() == 2: + exp_ms2.addSpectrum(spec) + _long_rename = {'rt': 'RT', 'intensity': 'inty'} + if not exp_ms1.empty(): + st.session_state["view_ms1"] = exp_ms1.get_df(long=True).rename(columns=_long_rename) + else: + st.session_state["view_ms1"] = pd.DataFrame(columns=['RT', 'mz', 'inty']) + if not exp_ms2.empty(): + st.session_state["view_ms2"] = exp_ms2.get_df(long=True).rename(columns=_long_rename) + else: + st.session_state["view_ms2"] = pd.DataFrame(columns=['RT', 'mz', 'inty']) + + +def plot_bpc_tic() -> go.Figure: + """Plot the base peak and total ion chromatogram (TIC). + + Returns: + A plotly Figure object containing the BPC and TIC plot. + """ + fig = go.Figure() + max_int = 0 + if st.session_state.view_tic: + df = st.session_state.view_ms1.groupby("RT").sum().reset_index() + df["type"] = "TIC" + if df["inty"].max() > max_int: + max_int = df["inty"].max() + fig = df.plot( + backend="ms_plotly", + kind="chromatogram", + x="RT", + y="inty", + by="type", + color="#f24c5c", + show_plot=False, + grid=False, + aggregate_duplicates=True, + ) + if st.session_state.view_bpc: + df = st.session_state.view_ms1.groupby("RT").max().reset_index() + df["type"] = "BPC" + if df["inty"].max() > max_int: + max_int = df["inty"].max() + fig = df.plot( + backend="ms_plotly", + kind="chromatogram", + x="RT", + y="inty", + by="type", + color="#2d3a9d", + show_plot=False, + grid=False, + aggregate_duplicates=True, + ) + if st.session_state.view_eic: + df = st.session_state.view_ms1 + target_value = st.session_state.view_eic_mz.strip().replace(",", ".") + try: + target_value = float(target_value) + ppm_tolerance = st.session_state.view_eic_ppm + tolerance = (target_value * ppm_tolerance) / 1e6 + + # Filter the DataFrame + df_eic = df[ + (df["mz"] >= target_value - tolerance) + & (df["mz"] <= target_value + tolerance) + ].copy() + if not df_eic.empty: + df_eic.loc[:, "type"] = "XIC" + if df_eic["inty"].max() > max_int: + max_int = df_eic["inty"].max() + fig = df_eic.plot( + backend="ms_plotly", + kind="chromatogram", + x="RT", + y="inty", + by="type", + color="#f6bf26", + show_plot=False, + grid=False, + aggregate_duplicates=True, + ) + except ValueError: + st.error("Invalid m/z value for XIC provided. Please enter a valid number.") + + fig.update_yaxes(range=[0, max_int]) + fig.update_layout( + title=f"{st.session_state.view_selected_file}", + xaxis_title="retention time (s)", + yaxis_title="intensity", + plot_bgcolor="rgb(255,255,255)", + height=500, + ) + fig.layout.template = "plotly_white" + return fig + + +@st.cache_resource +def plot_ms_spectrum(df, title, bin_peaks, num_x_bins): + fig = df.plot( + kind="spectrum", + backend="ms_plotly", + x="mz", + y="intensity", + color="#2d3a9d", + title=title, + show_plot=False, + grid=False, + bin_peaks=bin_peaks, + num_x_bins=num_x_bins, + aggregate_duplicates=True, + ) + fig.update_layout( + template="plotly_white", dragmode="select", plot_bgcolor="rgb(255,255,255)" + ) + return fig + + +@st.fragment +def view_peak_map(): + df = st.session_state.view_ms1 + if "view_peak_map_selection" in st.session_state: + box = st.session_state.view_peak_map_selection.selection.box + if box: + df = st.session_state.view_ms1.copy() + df = df[df["RT"] > box[0]["x"][0]] + df = df[df["mz"] > box[0]["y"][1]] + df = df[df["mz"] < box[0]["y"][0]] + df = df[df["RT"] < box[0]["x"][1]] + if len(df) == 0: + return + peak_map = df.plot( + kind="peakmap", + x="RT", + y="mz", + z="inty", + title=st.session_state.view_selected_file, + grid=False, + show_plot=False, + bin_peaks=True, + backend="ms_plotly", + aggregate_duplicates=True, + ) + peak_map.update_layout(template="simple_white", dragmode="select") + c1, c2 = st.columns(2) + with c1: + st.info( + "💡 Zoom in via rectangular selection for more details and 3D plot. Double click plot to zoom back out." + ) + show_fig( + peak_map, + f"peak_map_{st.session_state.view_selected_file}", + selection_session_state_key="view_peak_map_selection", + ) + with c2: + if df.shape[0] < 2500: + peak_map_3D = df.plot( + kind="peakmap", + plot_3d=True, + backend="ms_plotly", + x="RT", + y="mz", + z="inty", + zlabel="Intensity", + title="", + show_plot=False, + grid=False, + bin_peaks=st.session_state.spectrum_bin_peaks, + num_x_bins=st.session_state.spectrum_num_bins, + height=650, + width=900, + aggregate_duplicates=True, + ) + st.plotly_chart(peak_map_3D, use_container_width=True) + + +@st.fragment +def view_spectrum(): + cols = st.columns([0.34, 0.66]) + with cols[0]: + df = st.session_state.view_spectra.copy() + df["spectrum ID"] = df.index + 1 + index = display_large_dataframe( + df, + column_order=[ + "spectrum ID", + "RT", + "MS level", + "max intensity m/z", + "precursor m/z", + ], + selection_mode="single-row", + on_select="rerun", + use_container_width=True, + hide_index=True, + ) + with cols[1]: + if (index is not None) and (len(df) != 0): + df = st.session_state.view_spectra.iloc[index] + if "view_spectrum_selection" in st.session_state: + box = st.session_state.view_spectrum_selection.selection.box + if box: + mz_min, mz_max = sorted(box[0]["x"]) + mask = (df["mzarray"] > mz_min) & (df["mzarray"] < mz_max) + df["intarray"] = df["intarray"][mask] + df["mzarray"] = df["mzarray"][mask] + + if df["mzarray"].size > 0: + title = f"{st.session_state.view_selected_file} spec={index+1} mslevel={df['MS level']}" + if df["precursor m/z"] > 0: + title += f" precursor m/z: {round(df['precursor m/z'], 4)}" + + df_selected = pd.DataFrame( + { + "mz": df["mzarray"], + "intensity": df["intarray"], + } + ) + df_selected["RT"] = df["RT"] + df_selected["MS level"] = df["MS level"] + df_selected["precursor m/z"] = df["precursor m/z"] + df_selected["max intensity m/z"] = df["max intensity m/z"] + + fig = plot_ms_spectrum( + df_selected, + title, + st.session_state.spectrum_bin_peaks, + st.session_state.spectrum_num_bins, + ) + + show_fig(fig, title.replace(" ", "_"), True, "view_spectrum_selection") + else: + st.session_state.pop("view_spectrum_selection") + st.rerun() + else: + st.info("💡 Select rows in the spectrum table to display plot.") + + +@st.fragment() +def view_bpc_tic(): + cols = st.columns(5) + cols[0].checkbox( + "Total Ion Chromatogram (TIC)", True, key="view_tic", help="Plot TIC." + ) + cols[1].checkbox( + "Base Peak Chromatogram (BPC)", True, key="view_bpc", help="Plot BPC." + ) + cols[2].checkbox( + "Extracted Ion Chromatogram (EIC/XIC)", + True, + key="view_eic", + help="Plot extracted ion chromatogram with specified m/z.", + ) + cols[3].text_input( + "XIC m/z", + "235.1189", + help="m/z for XIC calculation.", + key="view_eic_mz", + ) + cols[4].number_input( + "XIC ppm tolerance", + 0.1, + 50.0, + 10.0, + 1.0, + help="Tolerance for XIC calculation (ppm).", + key="view_eic_ppm", + ) + fig = plot_bpc_tic() + show_fig(fig, f"BPC-TIC-{st.session_state.view_selected_file}") diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 6a587cd..6479cb0 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -5,7 +5,7 @@ import threading from pathlib import Path from .Logger import Logger -from .ParameterManager import ParameterManager +from .ParameterManager import ParameterManager, bool_param_paths_from_param_xml_ini import sys import importlib.util import json @@ -268,10 +268,14 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool # Load merged parameters (_defaults + user overrides) for this tool instance merged_params = self.parameter_manager.get_merged_params(params_key) + + # Load flag parameter names: params.json takes priority (survives session restart), + # session_state is the live fallback during the current session. flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {}) - if not flag_map: - flag_map = st.session_state.get("_topp_flag_params", {}) - flag_params = set(flag_map.get(params_key, [])) + flag_list = flag_map.get(params_key) + if flag_list is None: + flag_list = st.session_state.get("_topp_flag_params", {}).get(params_key, []) + flag_params: set = set(flag_list) # Construct commands for each process for i in range(n_processes): @@ -294,54 +298,45 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool # Add merged TOPP tool parameters (_defaults + user overrides) for k, v in merged_params.items(): if k in flag_params: - # CLI flag: include "-k" only when enabled + # CLI flag: include "-k" only when truthy, omit when false if isinstance(v, str): - is_enabled = v.lower() in {"true", "1", "yes", "on"} + is_enabled = v.lower() == "true" else: is_enabled = bool(v) if is_enabled: command += [f"-{k}"] continue - # For non-flag parameters, skip entirely if empty. - # Note: 0 and 0.0 are valid values, so use explicit checks. - if v == "" or v is None: + # Regular parameter: skip empty/None/empty-list, append value otherwise + if v == "" or v is None or (isinstance(v, list) and not v): continue command += [f"-{k}"] if isinstance(v, str) and "\n" in v: command += v.split("\n") - elif isinstance(v, bool): - command += [str(v).lower()] + elif isinstance(v, list): + command += [str(x) for x in v] else: command += [str(v)] # Add custom parameters for k, v in custom_params.items(): if k in flag_params: if isinstance(v, str): - is_enabled = v.lower() in {"true", "1", "yes", "on"} + is_enabled = v.lower() == "true" else: is_enabled = bool(v) if is_enabled: command += [f"-{k}"] continue - if v == "" or v is None: + if v == "" or v is None or (isinstance(v, list) and not v): continue command += [f"-{k}"] if isinstance(v, list): command += [str(x) for x in v] - elif isinstance(v, bool): - command += [str(v).lower()] else: command += [str(v)] # Add threads parameter for TOPP tools command += ["-threads", str(threads_per_command)] commands.append(command) - for idx, cmd in enumerate(commands): - # Print list-form command joined into a single string for readability - print(f" 🔹 Command {idx + 1}: {' '.join(cmd)}") - print("==========================================================\n") - - # Run command(s) if len(commands) == 1: return self.run_command(commands[0]) diff --git a/src/workflow/ParameterManager.py b/src/workflow/ParameterManager.py index 19e8700..2838c1b 100644 --- a/src/workflow/ParameterManager.py +++ b/src/workflow/ParameterManager.py @@ -3,8 +3,52 @@ import shutil import subprocess import streamlit as st +import xml.etree.ElementTree as ET from pathlib import Path + +def bool_param_paths_from_param_xml_ini(ini_path: Path, tool_stem: str) -> set[str]: + """ + Return short parameter paths for every ```` in a ParamXML .ini file. + + Paths match the suffix after ``Tool:1:`` in pyOpenMS (e.g. ``algorithm:epd:masstrace_snr_filtering``). + """ + try: + root = ET.parse(ini_path).getroot() + except (ET.ParseError, OSError): + return set() + + def local_tag(el: ET.Element) -> str: + t = el.tag + return t.rsplit("}", 1)[-1] if isinstance(t, str) and "}" in t else str(t) + + out: set[str] = set() + + def walk(el: ET.Element, parts: tuple[str, ...]) -> None: + for ch in el: + lt = local_tag(ch) + if lt == "NODE": + nm = ch.get("name") or "" + walk(ch, parts + (nm,)) + elif lt == "ITEM" and (ch.get("type") or "").lower() == "bool": + nm = ch.get("name") or "" + segs = [p for p in parts if p] + if nm: + segs.append(nm) + if not segs: + continue + # Strip tool root NODE name and instance NODE "1" (not part of pyOpenMS short keys) + while segs and segs[0] in (tool_stem, "1"): + segs.pop(0) + if segs: + out.add(":".join(segs)) + + for ch in root: + if local_tag(ch) == "NODE": + walk(ch, ()) + return out + + class ParameterManager: """ Manages the parameters for a workflow, including saving parameters to a JSON file, @@ -29,6 +73,29 @@ def __init__(self, workflow_dir: Path, workflow_name: str = None): # Store workflow name for preset loading; default to directory stem if not provided self.workflow_name = workflow_name or workflow_dir.stem + def bool_pairs_session_key(self) -> str: + """Session state key holding a set of (tool name, param path) for bool TOPP params.""" + return f"{self.ini_dir.parent.stem}-topp-bool-pairs" + + def get_bool_param_pairs(self) -> set: + """Return the cached set of (tool, param path) bool params; empty set if none.""" + return st.session_state.get(self.bool_pairs_session_key(), set()) + + def _merge_bool_params_from_ini(self, tool: str) -> None: + """Load tool.ini (XML) and merge type=bool parameter paths into session_state.""" + ini_path = Path(self.ini_dir, f"{tool}.ini") + if not ini_path.exists(): + return + try: + sk = self.bool_pairs_session_key() + if sk not in st.session_state: + st.session_state[sk] = set() + for short in bool_param_paths_from_param_xml_ini(ini_path, tool): + st.session_state[sk].add((tool, short)) + except RuntimeError: + # No Streamlit session (e.g. plain `python` import) + pass + def create_ini(self, tool: str) -> bool: """ Create an ini file for a TOPP tool if it doesn't exist. @@ -41,11 +108,14 @@ def create_ini(self, tool: str) -> bool: """ ini_path = Path(self.ini_dir, tool + ".ini") if ini_path.exists(): + self._merge_bool_params_from_ini(tool) return True try: subprocess.call([tool, "-write_ini", str(ini_path)]) except FileNotFoundError: return False + if ini_path.exists(): + self._merge_bool_params_from_ini(tool) return ini_path.exists() def save_parameters(self) -> None: @@ -65,7 +135,7 @@ def save_parameters(self) -> None: # Advanced parameters are only in session state if the view is active json_params = self.get_parameters_from_json() | json_params - # get a list of TOPP tools which are in session state + # get a list of TOPP tools (or tool instance names) which are in session state current_topp_tools = list( set( [ @@ -75,12 +145,16 @@ def save_parameters(self) -> None: ] ) ) - # for each TOPP tool, open the ini file + # Retrieve the instance-name → real-tool-name mapping (set by input_TOPP) + tool_instance_map = st.session_state.get("_topp_tool_instance_map", {}) + # for each TOPP tool (or instance name), open the ini file for tool in current_topp_tools: - if not self.create_ini(tool): + # Resolve instance name to real tool name for create_ini / ini loading + real_tool = tool_instance_map.get(tool, tool) + if not self.create_ini(real_tool): # Could not create ini file - skip this tool continue - ini_path = Path(self.ini_dir, f"{tool}.ini") + ini_path = Path(self.ini_dir, f"{real_tool}.ini") if tool not in json_params: json_params[tool] = {} # load the param object @@ -92,19 +166,26 @@ def save_parameters(self) -> None: # Skip display keys used by multiselect widgets if key.endswith("_display"): continue - # get ini_key - ini_key = key.replace(self.topp_param_prefix, "").encode() + # get ini_key – map instance name back to real tool name + ini_key = key.replace(self.topp_param_prefix, "") + if tool != real_tool: + ini_key = ini_key.replace(f"{tool}:1:", f"{real_tool}:1:", 1) + ini_key = ini_key.encode() # get ini (default) value by ini_key ini_value = param.getValue(ini_key) is_list_param = isinstance(ini_value, list) - # check if value is different from default OR is an empty list parameter + # Effective default: _defaults value if present, else ini value + short_key = key.split(":1:")[1] + defaults = json_params.get("_defaults", {}).get(tool, {}) + default_value = defaults.get(short_key, ini_value) + # check if value is different from effective default OR is an empty list parameter if ( - (ini_value != value) - or (key.split(":1:")[1] in json_params[tool]) + (default_value != value) + or (short_key in json_params[tool]) or (is_list_param and not value) # Always save empty list params ): # store non-default value - json_params[tool][key.split(":1:")[1]] = value + json_params[tool][short_key] = value # Save to json file with open(self.params_file, "w", encoding="utf-8") as f: json.dump(json_params, f, indent=4) @@ -129,7 +210,7 @@ def get_parameters_from_json(self) -> dict: except: st.error("**ERROR**: Attempting to load an invalid JSON parameter file. Reset to defaults.") return {} - + def get_merged_params(self, tool_instance_name: str, ini_params: dict = None) -> dict: """ Three-layer parameter merge: ini defaults < _defaults < user overrides. @@ -154,17 +235,20 @@ def get_merged_params(self, tool_instance_name: str, ini_params: dict = None) -> merged.update(user) return merged - def get_topp_parameters(self, tool: str) -> dict: + def get_topp_parameters(self, tool: str, tool_instance_name: str = None) -> dict: """ Get all parameters for a TOPP tool, merging defaults with user values. Args: - tool: Name of the TOPP tool (e.g., "CometAdapter") + tool: Name of the TOPP tool executable (e.g., "CometAdapter") + tool_instance_name: Optional instance name used for parameter storage + (e.g., "IDFilter_step1"). If not provided, defaults to tool name. Returns: Dict with parameter names as keys (without tool prefix) and their values. Returns empty dict if ini file doesn't exist. """ + instance_name = tool_instance_name or tool ini_path = Path(self.ini_dir, f"{tool}.ini") if not ini_path.exists(): return {} @@ -175,18 +259,14 @@ def get_topp_parameters(self, tool: str) -> dict: # Build dict from ini (extract short key names) prefix = f"{tool}:1:" - full_params = {} + ini_params = {} for key in param.keys(): key_str = key.decode() if isinstance(key, bytes) else str(key) if prefix in key_str: short_key = key_str.split(prefix, 1)[1] - full_params[short_key] = param.getValue(key) - - # Override with user-modified values from JSON - user_params = self.get_parameters_from_json().get(tool, {}) - full_params.update(user_params) + ini_params[short_key] = param.getValue(key) - return full_params + return self.get_merged_params(instance_name, ini_params=ini_params) def reset_to_default_parameters(self) -> None: """ diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index befde2e..d426e3a 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -23,6 +23,32 @@ from src.workflow._log_status import classify_log_outcome +def _mounted_data_root() -> Union[Path, None]: + """Return the validated mount root from the ``local_data_dir`` setting. + + The browser renders only when ``local_data_dir`` is an actual mount + point inside the container — i.e. the operator passed ``-v`` / + ``--bind`` / ``volumeMount`` to attach host data. Existence alone is + no longer sufficient because the image now pre-creates the path so + apptainer/singularity binds have a real attach target; without + ``os.path.ismount`` the browser would render an empty tree for every + user who didn't mount anything. + """ + settings = st.session_state.get("settings") or {} + raw = (settings.get("local_data_dir") or "").strip() + if not raw: + return None + try: + p = Path(raw).expanduser().resolve(strict=True) + except (OSError, RuntimeError): + return None + if not p.is_dir(): + return None + if not os.path.ismount(p): + return None + return p + + class StreamlitUI: """ Provides an interface for Streamlit applications to handle file uploads, @@ -77,6 +103,8 @@ def upload_widget( c1, c2 = st.columns(2) c1.markdown("**Upload file(s)**") + mount_root = _mounted_data_root() if st.session_state.location == "online" else None + if st.session_state.location == "local": c2_text, c2_checkbox = c2.columns([1.5, 1], gap="large") c2_text.markdown("**OR add files from local folder**") @@ -247,7 +275,19 @@ def upload_widget( "This means that the original files will be used instead. " ) - if fallback and not any([f for f in Path(files_dir).iterdir() if f.name != "external_files.txt"]): + if mount_root is not None: + with c2: + self._mounted_drive_browser(key, name, file_types, files_dir, mount_root) + + external_files_path = Path(files_dir, "external_files.txt") + has_real_files = any( + p.name != "external_files.txt" for p in files_dir.iterdir() + ) + has_external_picks = external_files_path.exists() and any( + line.strip() and os.path.exists(line.strip()) + for line in external_files_path.read_text().splitlines() + ) + if fallback and not has_real_files and not has_external_picks: if isinstance(fallback, str): fallback = [fallback] for f in fallback: @@ -303,6 +343,179 @@ def upload_widget( elif not fallback: st.warning(f"No **{name}** files!") + def _resolve_browser_cwd(self, key: str, mount_root: Path) -> Path: + """Read cwd for this widget from session state, confine it to mount_root.""" + sess_key = f"mounted_cwd_{key}" + raw = st.session_state.get(sess_key, str(mount_root)) + try: + cwd = Path(raw).expanduser().resolve(strict=True) + except (OSError, RuntimeError): + cwd = mount_root + if cwd != mount_root and mount_root not in cwd.parents: + cwd = mount_root + st.session_state[sess_key] = str(cwd) + return cwd + + def _mounted_drive_browser( + self, + key: str, + name: str, + file_types: List[str], + files_dir: Path, + mount_root: Path, + ) -> None: + """Render a tree browser for a mounted host directory. + + Selected files are referenced in place via ``external_files.txt`` — + the same mechanism the offline tkinter flow uses. + """ + external_files = Path(files_dir, "external_files.txt") + if not external_files.exists(): + external_files.touch() + + cwd = self._resolve_browser_cwd(key, mount_root) + sess_cwd_key = f"mounted_cwd_{key}" + + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + + with st.container(border=True): + st.markdown( + f"**Add {name} files from mounted directory** " + f"`{mount_root}`" + ) + + # Breadcrumbs: compact tertiary buttons separated by », + # with a right-aligned Parent button. + try: + rel = cwd.relative_to(mount_root) + segments = [mount_root.name] + list(rel.parts) if rel.parts else [mount_root.name] + except ValueError: + segments = [mount_root.name] + n = len(segments) + ratios: List[float] = [] + for i in range(n): + ratios.append(max(len(segments[i]), 3)) + if i < n - 1: + ratios.append(1) + ratios.append(20) # flexible spacer + ratios.append(6) # parent button slot + crumb_cols = st.columns(ratios, vertical_alignment="center") + col_idx = 0 + for i, seg in enumerate(segments): + target = mount_root.joinpath(*segments[1 : i + 1]) if i > 0 else mount_root + if crumb_cols[col_idx].button( + seg, + key=f"crumb_{key}_{i}", + type="tertiary", + ): + st.session_state[sess_cwd_key] = str(target) + st.rerun(scope="fragment") + col_idx += 1 + if i < n - 1: + crumb_cols[col_idx].markdown( + "»", + unsafe_allow_html=True, + ) + col_idx += 1 + # spacer column + col_idx += 1 + if cwd != mount_root: + if crumb_cols[col_idx].button( + "⬆ Parent", + key=f"mounted_parent_{key}", + type="tertiary", + ): + st.session_state[sess_cwd_key] = str(cwd.parent) + st.rerun(scope="fragment") + + try: + entries = sorted( + (p for p in cwd.iterdir() if not p.name.startswith(".")), + key=lambda p: (not p.is_dir(), p.name.lower()), + ) + except PermissionError: + st.error(f"Permission denied reading `{cwd}`.") + return + + def _is_match(p: Path) -> bool: + return any(p.name.endswith(f".{ft}") for ft in file_types) + + subdirs = [p for p in entries if p.is_dir() and not _is_match(p)] + bundled = [p for p in entries if p.is_dir() and _is_match(p)] + files = [p for p in entries if p.is_file() and _is_match(p)] + + for d in subdirs: + indent, body = st.columns([1, 60], vertical_alignment="center") + if body.button( + f"📂 {d.name}/", + key=f"mounted_dir_{key}_{d.name}", + type="tertiary", + ): + st.session_state[sess_cwd_key] = str(d) + st.rerun(scope="fragment") + + selectable = bundled + files + selected_paths: List[str] = [] + for f in selectable: + cb_key = f"mounted_pick_{key}_{f}" + size_label = "" + if f.is_file(): + try: + size_mb = f.stat().st_size / (1024 * 1024) + size_label = f" · {size_mb:.1f} MB" + except OSError: + pass + icon = "🗂️" if f.is_dir() else "📄" + if st.checkbox( + f"{icon} {f.name}{size_label}", + key=cb_key, + ): + selected_paths.append(str(f)) + + if not subdirs and not selectable: + st.info( + f"No subdirectories or files matching " + f"**{', '.join('.' + ft for ft in file_types)}** here." + ) + + count = len(selected_paths) + if st.button( + f"➕ Add {count} selected {name} file(s)" if count else f"➕ Add selected {name} file(s)", + key=f"mounted_add_{key}", + type="primary", + use_container_width=True, + disabled=count == 0, + ): + existing = set( + line.strip() + for line in external_files.read_text().splitlines() + if line.strip() + ) + added = 0 + with open(external_files, "a") as fh: + for p in selected_paths: + if p not in existing: + fh.write(f"{p}\n") + existing.add(p) + added += 1 + # Clear the checkboxes by removing their session keys. + for f in selectable: + st.session_state.pop(f"mounted_pick_{key}_{f}", None) + st.success(f"Added {added} file(s) from `{cwd}`.") + st.rerun(scope="fragment") + def select_input_file( self, key: str, @@ -606,7 +819,6 @@ def format_files(input: Any) -> List[str]: self.parameter_manager.save_parameters() - @st.fragment def input_TOPP( self, topp_tool_name: str, @@ -619,6 +831,7 @@ def input_TOPP( display_subsection_tabs: bool = False, custom_defaults: dict = {}, tool_instance_name: str = None, + reactive: bool = False, ) -> None: """ Generates input widgets for TOPP tool parameters dynamically based on the tool's @@ -642,7 +855,58 @@ def input_TOPP( defaults to topp_tool_name. The instance name is used for session state keys and parameter storage, while topp_tool_name is used for the actual tool executable and ini file creation. + reactive (bool, optional): If True, widget changes trigger the parent + section to re-render, enabling conditional UI based on this widget's + value. Use when downstream UI depends on a parameter value (e.g., + TMT type driving channel count). Default is False. """ + if reactive: + self._input_TOPP_impl( + topp_tool_name, num_cols, exclude_parameters, include_parameters, + flag_parameters, display_tool_name, display_subsections, + display_subsection_tabs, custom_defaults, tool_instance_name, + ) + else: + self._input_TOPP_fragmented( + topp_tool_name, num_cols, exclude_parameters, include_parameters, + flag_parameters, display_tool_name, display_subsections, + display_subsection_tabs, custom_defaults, tool_instance_name, + ) + + @st.fragment + def _input_TOPP_fragmented( + self, + topp_tool_name: str, + num_cols: int = 4, + exclude_parameters: List[str] = [], + include_parameters: List[str] = [], + flag_parameters: List[str] = [], + display_tool_name: bool = True, + display_subsections: bool = True, + display_subsection_tabs: bool = False, + custom_defaults: dict = {}, + tool_instance_name: str = None, + ) -> None: + self._input_TOPP_impl( + topp_tool_name, num_cols, exclude_parameters, include_parameters, + flag_parameters, display_tool_name, display_subsections, + display_subsection_tabs, custom_defaults, tool_instance_name, + ) + + def _input_TOPP_impl( + self, + topp_tool_name: str, + num_cols: int = 4, + exclude_parameters: List[str] = [], + include_parameters: List[str] = [], + flag_parameters: List[str] = [], + display_tool_name: bool = True, + display_subsections: bool = True, + display_subsection_tabs: bool = False, + custom_defaults: dict = {}, + tool_instance_name: str = None, + ) -> None: + """Internal implementation of input_TOPP - contains all the widget logic.""" # Default instance name to the tool name when not provided if tool_instance_name is None: tool_instance_name = topp_tool_name @@ -651,16 +915,18 @@ def input_TOPP( if "_topp_tool_instance_map" not in st.session_state: st.session_state["_topp_tool_instance_map"] = {} st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name + + # Persist flag_parameters to session_state and params.json so run_topp + # can skip appending a value for these boolean CLI flags. if "_topp_flag_params" not in st.session_state: st.session_state["_topp_flag_params"] = {} st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters) - # Persist flag metadata so execution still sees it outside UI reruns/session context. - params = self.parameter_manager.get_parameters_from_json() - if "_flag_params" not in params: - params["_flag_params"] = {} - params["_flag_params"][tool_instance_name] = list(flag_parameters) - with open(self.parameter_manager.params_file, "w", encoding="utf-8") as f: - json.dump(params, f, indent=4) + _fp = self.parameter_manager.get_parameters_from_json() + if "_flag_params" not in _fp: + _fp["_flag_params"] = {} + _fp["_flag_params"][tool_instance_name] = list(flag_parameters) + with open(self.parameter_manager.params_file, "w", encoding="utf-8") as _f: + json.dump(_fp, _f, indent=4) if not display_subsections: display_subsection_tabs = False @@ -752,7 +1018,6 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: ":".join(key.decode().split(":")[:-1]) ), } - p["is_flag"] = (b"flag" in param.getTags(key)) # Parameter sections and subsections as string (e.g. "section:subsection") if display_subsections: p["sections"] = ":".join( @@ -843,43 +1108,16 @@ def display_TOPP_params(params: dict, num_cols): # sometimes strings with newline, handle as list if isinstance(p["value"], str) and "\n" in p["value"]: p["value"] = p["value"].split("\n") - # no-value CLI flag parameters should be shown as checkboxes - if p.get("is_flag", False): - flag_default = p["value"] - if isinstance(flag_default, str): - flag_default = flag_default.lower() in {"true", "1", "yes", "on"} - else: - flag_default = bool(flag_default) - # Streamlit widget keys persist in session_state and can override - # updated custom_defaults. Normalize and seed key explicitly. - if key in st.session_state: - current = st.session_state[key] - if isinstance(current, str): - st.session_state[key] = current.lower() in {"true", "1", "yes", "on"} - else: - st.session_state[key] = bool(current) - else: - st.session_state[key] = flag_default - cols[i].selectbox( - name, - options=[True, False], - index=0 if st.session_state[key] else 1, - format_func=lambda x: "True" if x else "False", - help=p["description"], - key=key, - ) # bools - elif isinstance(p["value"], bool): - bool_value = ( - (p["value"] == "true") - if type(p["value"]) == str - else p["value"] - ) - cols[i].selectbox( + if isinstance(p["value"], bool): + cols[i].markdown("##") + cols[i].checkbox( name, - options=[True, False], - index=0 if bool_value else 1, - format_func=lambda x: "True" if x else "False", + value=( + (p["value"] == "true") + if type(p["value"]) == str + else p["value"] + ), help=p["description"], key=key, ) @@ -964,6 +1202,7 @@ def on_multiselect_change(dk=display_key, tk=key): cols[i].error(f"Error in parameter **{p['name']}**.") print('Error parsing "' + p["name"] + '": ' + str(e)) + for section, params in param_sections.items(): if tabs is None: show_subsection_header(section, display_subsections) @@ -1435,7 +1674,8 @@ def remove_full_paths(d: dict) -> dict: general = {} for k, v in params.items(): - # skip if v is a file path + if k == "_defaults": + continue if isinstance(v, dict): topp[k] = v elif ".py" in k: @@ -1446,6 +1686,13 @@ def remove_full_paths(d: dict) -> dict: else: general[k] = v + # Merge _defaults into topp so summary shows custom defaults + user overrides + defaults = params.get("_defaults", {}) + for tool_name, default_vals in defaults.items(): + if tool_name not in topp: + topp[tool_name] = {} + topp[tool_name] = {**default_vals, **topp.get(tool_name, {})} + markdown = [] def dict_to_markdown(d: dict): diff --git a/src/workflow/WorkflowManager.py b/src/workflow/WorkflowManager.py index 302b079..856ae56 100644 --- a/src/workflow/WorkflowManager.py +++ b/src/workflow/WorkflowManager.py @@ -206,10 +206,9 @@ def stop_workflow(self) -> bool: return self._stop_local_workflow() def _stop_local_workflow(self) -> bool: - """Stop locally running workflow process - Windows Compatible""" + """Stop locally running workflow process""" import os import signal - import platform pid_dir = self.executor.pid_dir if not pid_dir.exists(): @@ -219,18 +218,11 @@ def _stop_local_workflow(self) -> bool: for pid_file in pid_dir.iterdir(): try: pid = int(pid_file.name) - # Windows - if platform.system() == "Windows": - os.system(f"taskkill /F /T /PID {pid}") - else: - # Linux/macOS - os.kill(pid, signal.SIGTERM) - + os.kill(pid, signal.SIGTERM) pid_file.unlink() stopped = True - except (ValueError, ProcessLookupError, PermissionError, OSError): - if pid_file.exists(): - pid_file.unlink() + except (ValueError, ProcessLookupError, PermissionError): + pid_file.unlink() # Clean up stale PID file # Clean up the pid directory shutil.rmtree(pid_dir, ignore_errors=True) diff --git a/test.py b/test.py new file mode 100644 index 0000000..8a2a3ad --- /dev/null +++ b/test.py @@ -0,0 +1,24 @@ +# test_my_math.py +import unittest +from urllib.request import urlretrieve + +from src.simpleworkflow import generate_random_table +from src.mzmlfileworkflow import mzML_file_get_num_spectra + +from pathlib import Path + +class TestSimpleWorkflow(unittest.TestCase): + def test_workflow(self): + result = generate_random_table(2, 3).shape + self.assertEqual(result, (2,3), "Expected dataframe shape.") + +class TestComplexWorkflow(unittest.TestCase): + def test_workflow(self): + # load data from url + urlretrieve("https://raw.githubusercontent.com/OpenMS/streamlit-template/main/example-data/mzML/Treatment.mzML", "testfile.mzML") + result = mzML_file_get_num_spectra("testfile.mzML") + Path("testfile.mzML").unlink() + self.assertEqual(result, 786, "Expected dataframe shape.") + +if __name__ == '__main__': + unittest.main() diff --git a/test_gui.py b/test_gui.py index 0ab2711..0485bae 100644 --- a/test_gui.py +++ b/test_gui.py @@ -1,40 +1,141 @@ -import json - -import pytest from streamlit.testing.v1 import AppTest +import pytest +from src import fileupload +import json +from pathlib import Path +import shutil -# Pages that AppTest.from_file can load in isolation. Pages using st.page_link -# require streamlit's navigation context (only set up when app.py runs), so they -# are covered indirectly by test_app_loads below. -DIRECTLY_TESTABLE_PAGES = [ - "content/workflow_fileupload.py", - "content/workflow_configure.py", - "content/workflow_run.py", - "content/results_library.py", - "content/results_proteomicslfq.py", -] +@pytest.fixture +def launch(request): + test = AppTest.from_file(request.param) -def _init(apptest): + ## Initialize session state ## with open("settings.json", "r") as f: - apptest.session_state.settings = json.load(f) - apptest.session_state.settings["test"] = True - apptest.secrets["workspace"] = "test" - return apptest + test.session_state.settings = json.load(f) + test.session_state.settings["test"] = True + test.secrets["workspace"] = "test" + return test -@pytest.fixture -def launch(request): - return _init(AppTest.from_file(request.param)) +# Test launching of all pages +@pytest.mark.parametrize( + "launch", + ( + # "content/quickstart.py", # NOTE: this page does not work due to streamlit.errors.StreamlitPageNotFoundError error + "content/documentation.py", + "content/topp_workflow_file_upload.py", + "content/topp_workflow_parameter.py", + "content/topp_workflow_execution.py", + "content/topp_workflow_results.py", + "content/file_upload.py", + "content/raw_data_viewer.py", + "content/run_example_workflow.py", + "content/download_section.py", + "content/simple_workflow.py", + "content/run_subprocess.py", + ), + indirect=True, +) +def test_launch(launch): + """Test if all pages can be launched without errors.""" + launch.run(timeout=30) # Increased timeout from 10 to 30 seconds + assert not launch.exception + + +########### PAGE SPECIFIC TESTS ############ +@pytest.mark.parametrize( + "launch,selection", + [ + ("content/documentation.py", "User Guide"), + ("content/documentation.py", "Installation"), + ( + "content/documentation.py", + "Developers Guide: How to build app based on this template", + ), + ("content/documentation.py", "Developers Guide: TOPP Workflow Framework"), + ("content/documentation.py", "Developer Guide: Windows Executables"), + ("content/documentation.py", "Developers Guide: Deployment"), + ("content/documentation.py", "Developers Guide: Kubernetes Deployment"), + ], + indirect=["launch"], +) +def test_documentation(launch, selection): + launch.run() + launch.selectbox[0].select(selection).run() + assert not launch.exception + + +@pytest.mark.parametrize("launch", ["content/file_upload.py"], indirect=True) +def test_file_upload_load_example(launch): + launch.run() + for i in launch.tabs: + if i.label == "Example Data": + i.button[0].click().run() + assert not launch.exception + + +# NOTE: All tabs are automatically checked +@pytest.mark.parametrize( + "launch,example", + [ + ("content/raw_data_viewer.py", "Blank.mzML"), + ("content/raw_data_viewer.py", "Treatment.mzML"), + ("content/raw_data_viewer.py", "Pool.mzML"), + ("content/raw_data_viewer.py", "Control.mzML"), + ], + indirect=["launch"], +) +def test_view_raw_ms_data(launch, example): + launch.run(timeout=30) # Increased timeout from 10 to 30 seconds + + ## Load Example file, based on implementation of fileupload.load_example_mzML_files() ### + mzML_dir = Path(launch.session_state.workspace, "mzML-files") + # Copy files from example-data/mzML to workspace mzML directory, add to selected files + for f in Path("example-data", "mzML").glob("*.mzML"): + try: + shutil.copy(f, mzML_dir) + except shutil.SameFileError: + pass # File already exists as a symlink to the same source (on Linux) + launch.run() -@pytest.mark.parametrize("launch", DIRECTLY_TESTABLE_PAGES, indirect=True) -def test_page_loads(launch): - launch.run(timeout=30) + ## TODO: Figure out a way to select a spectrum to be displayed + launch.selectbox[0].select(example).run() assert not launch.exception -def test_app_loads(): - app = _init(AppTest.from_file("app.py")) - app.run(timeout=30) - assert not app.exception +@pytest.mark.parametrize( + "launch,example", + [ + ("content/run_example_workflow.py", ["Blank"]), + ("content/run_example_workflow.py", ["Treatment"]), + ("content/run_example_workflow.py", ["Pool"]), + ("content/run_example_workflow.py", ["Control"]), + ("content/run_example_workflow.py", ["Control", "Blank"]), + ], + indirect=["launch"], +) +def test_run_workflow(launch, example): + launch.run() + ## Load Example file, based on implementation of fileupload.load_example_mzML_files() ### + mzML_dir = Path(launch.session_state.workspace, "mzML-files") + + # Copy files from example-data/mzML to workspace mzML directory, add to selected files + for f in Path("example-data", "mzML").glob("*.mzML"): + try: + shutil.copy(f, mzML_dir) + except shutil.SameFileError: + pass # File already exists as a symlink to the same source (on Linux) + launch.run() + + ## Select experiments to process + for e in example: + launch.multiselect[0].select(e) + + launch.run() + assert not launch.exception + + # Press the "Run Workflow" button + launch.button[1].click().run(timeout=60) + assert not launch.exception diff --git a/tests/test_legal_links.py b/tests/test_legal_links.py new file mode 100644 index 0000000..a38e201 --- /dev/null +++ b/tests/test_legal_links.py @@ -0,0 +1,160 @@ +""" +Tests for get_legal_links() in src/common/common.py. + +get_legal_links() resolves the Impressum / Privacy Policy / Terms of Use URLs +shown in the sidebar footer (on every page) and the privacy-policy link wired +into the GDPR consent banner. It merges the optional "legal_links" object from +settings.json over the built-in official-OpenMS defaults so that: + + * apps built from a settings.json without a "legal_links" key still inherit + working legal links by default, + * a self-hosting fork can override any or all of the three URLs, + * an empty/blank override value never erases a default. + +Streamlit (and the other heavy runtime deps pulled in by common.py) are mocked +before import so the helper can be unit-tested without a running Streamlit app, +mirroring tests/test_parameter_presets.py. +""" +import os +import sys +from unittest.mock import MagicMock + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + + +class FakeSessionState(dict): + """Minimal stand-in for Streamlit's SessionState. + + Supports both attribute access (``state.settings``) and item/membership + access (``"settings" in state``), exactly like the real SessionState that + common.py relies on. + """ + + def __getattr__(self, name): + try: + return self[name] + except KeyError as exc: + raise AttributeError(name) from exc + + def __setattr__(self, name, value): + self[name] = value + + +# Mock streamlit (with a SessionState-like session_state) and the other heavy +# imports pulled in by src/common/common.py, so importing get_legal_links here +# doesn't require a running Streamlit app context. +# +# IMPORTANT: these mocks are installed into sys.modules only for the duration of +# the import below and then restored, so they don't leak into other test modules +# (e.g. the AppTest-based tests that need the real `streamlit` package). This +# mirrors the pattern in tests/test_parameter_presets.py. +mock_streamlit = MagicMock() +mock_streamlit.session_state = FakeSessionState() + +_MOCKED_MODULES = { + "streamlit": mock_streamlit, + "streamlit.components": MagicMock(), + "streamlit.components.v1": MagicMock(), + "streamlit.source_util": MagicMock(), + "pandas": MagicMock(), + "psutil": MagicMock(), + # Local submodules with their own heavy deps (e.g. the captcha image library). + "src.common.captcha_": MagicMock(), + "src.common.admin": MagicMock(), +} +_saved_modules = {name: sys.modules.get(name) for name in _MOCKED_MODULES} +sys.modules.update(_MOCKED_MODULES) + +# Force a FRESH import of src.common.common under the streamlit mock, even if an +# earlier test module (e.g. test_gui.py) already imported the real-streamlit-bound +# version. Save whatever was cached first so we can restore it afterwards. +_saved_common = sys.modules.pop("src.common.common", None) + +from src.common.common import get_legal_links, DEFAULT_LEGAL_LINKS # noqa: E402 + +# Restore the real modules (or remove ones that weren't present) so that other +# test modules get the genuine packages. +for _name, _orig in _saved_modules.items(): + if _orig is None: + sys.modules.pop(_name, None) + else: + sys.modules[_name] = _orig +# Restore the original cached common module (the real-streamlit-bound one, if +# any) so AppTest-based test modules keep getting the genuine package. +# get_legal_links keeps working: it holds a reference to the freshly-imported +# mock-bound module's globals (and the same `mock_streamlit` object the tests +# mutate). +if _saved_common is None: + sys.modules.pop("src.common.common", None) +else: + sys.modules["src.common.common"] = _saved_common + + +def setup_function(_): + """Reset session_state before each test for isolation.""" + mock_streamlit.session_state = FakeSessionState() + + +def test_defaults_point_to_openms(): + """The built-in defaults are the official OpenMS pages.""" + assert DEFAULT_LEGAL_LINKS == { + "impressum": "https://openms.de/impressum", + "privacy": "https://openms.de/privacy", + "terms": "https://openms.de/terms", + } + + +def test_defaults_when_settings_not_loaded(): + """No settings loaded at all -> defaults, no crash.""" + mock_streamlit.session_state = FakeSessionState() + assert get_legal_links() == DEFAULT_LEGAL_LINKS + + +def test_defaults_when_no_legal_links_key(): + """settings present but without 'legal_links' -> all OpenMS defaults.""" + mock_streamlit.session_state = FakeSessionState({"settings": {}}) + assert get_legal_links() == DEFAULT_LEGAL_LINKS + + +def test_overrides_replace_defaults(): + """A fork's custom legal_links replace every default.""" + mock_streamlit.session_state = FakeSessionState( + { + "settings": { + "legal_links": { + "impressum": "https://acme.example/impressum", + "privacy": "https://acme.example/privacy", + "terms": "https://acme.example/terms", + } + } + } + ) + assert get_legal_links() == { + "impressum": "https://acme.example/impressum", + "privacy": "https://acme.example/privacy", + "terms": "https://acme.example/terms", + } + + +def test_partial_override_keeps_other_defaults(): + """Overriding only one link leaves the others at their OpenMS default.""" + mock_streamlit.session_state = FakeSessionState( + {"settings": {"legal_links": {"impressum": "https://acme.example/impressum"}}} + ) + links = get_legal_links() + assert links["impressum"] == "https://acme.example/impressum" + assert links["privacy"] == DEFAULT_LEGAL_LINKS["privacy"] + assert links["terms"] == DEFAULT_LEGAL_LINKS["terms"] + + +def test_empty_or_none_override_falls_back_to_default(): + """A blank/None override must not erase the default for that key.""" + mock_streamlit.session_state = FakeSessionState( + {"settings": {"legal_links": {"privacy": "", "impressum": None}}} + ) + links = get_legal_links() + assert links["privacy"] == DEFAULT_LEGAL_LINKS["privacy"] + assert links["impressum"] == DEFAULT_LEGAL_LINKS["impressum"] + assert links["terms"] == DEFAULT_LEGAL_LINKS["terms"] diff --git a/tests/test_parameter_defaults.py b/tests/test_parameter_defaults.py new file mode 100644 index 0000000..9ccd679 --- /dev/null +++ b/tests/test_parameter_defaults.py @@ -0,0 +1,363 @@ +""" +Tests for get_merged_params() and the refactored get_topp_parameters(). + +This module verifies the three-layer parameter merge: + ini defaults < _defaults < user overrides +""" +import os +import sys +import json +import pytest +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# Mock streamlit before importing ParameterManager so that the imported module +# uses a controllable `st.session_state` (a plain dict) instead of the real one, +# which requires a running Streamlit app context. +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +# Temporarily replace streamlit in sys.modules so that ParameterManager's +# `import streamlit as st` picks up the mock. Restore immediately after import +# so other test files (e.g., test_gui.py AppTest) get the real streamlit. +_original_streamlit = sys.modules.get('streamlit') +sys.modules['streamlit'] = mock_streamlit + +from src.workflow.ParameterManager import ParameterManager + +if _original_streamlit is not None: + sys.modules['streamlit'] = _original_streamlit +else: + sys.modules.pop('streamlit', None) + +# Remove cached src.workflow modules that were imported with mocked streamlit so +# that AppTest (in test_gui.py) re-imports them fresh with the real package. +for _key in list(sys.modules.keys()): + if _key.startswith('src.workflow'): + sys.modules.pop(_key, None) + + +@pytest.fixture(autouse=True) +def reset_streamlit_state(): + """Reset mock streamlit session state before each test.""" + mock_streamlit.session_state.clear() + yield + + +@pytest.fixture +def temp_workflow_dir(): + """Create a temporary workflow directory for testing.""" + with tempfile.TemporaryDirectory() as tmpdir: + workflow_dir = Path(tmpdir) / "test-workflow" + workflow_dir.mkdir() + ini_dir = workflow_dir / "ini" + ini_dir.mkdir() + yield workflow_dir + + +class TestGetMergedParams: + """Tests for ParameterManager.get_merged_params().""" + + def test_returns_ini_params_when_no_json(self, temp_workflow_dir): + """ini params returned when params.json doesn't exist.""" + pm = ParameterManager(temp_workflow_dir) + ini_params = {"algorithm:param": 1.0, "algorithm:other": "hello"} + + result = pm.get_merged_params("SomeTool", ini_params=ini_params) + + assert result == {"algorithm:param": 1.0, "algorithm:other": "hello"} + + def test_defaults_override_ini(self, temp_workflow_dir): + """_defaults layer overrides ini values.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool", ini_params={"algorithm:param": 1.0}) + + assert result["algorithm:param"] == 42.0 + + def test_user_overrides_defaults(self, temp_workflow_dir): + """User overrides take priority over _defaults.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}}, + "SomeTool": {"algorithm:param": 99.0} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool", ini_params={"algorithm:param": 1.0}) + + assert result["algorithm:param"] == 99.0 + + def test_full_three_layer_merge(self, temp_workflow_dir): + """All three layers merge correctly: ini < _defaults < user.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": { + "SomeTool": { + "algorithm:param_a": 10.0, # overrides ini + "algorithm:param_b": 20.0, # overrides ini, NOT overridden by user + } + }, + "SomeTool": { + "algorithm:param_a": 99.0, # overrides _defaults + "algorithm:param_c": 55.0, # only in user + } + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + ini_params = { + "algorithm:param_a": 1.0, + "algorithm:param_b": 2.0, + "algorithm:param_d": 3.0, # only in ini + } + + result = pm.get_merged_params("SomeTool", ini_params=ini_params) + + assert result["algorithm:param_a"] == 99.0 # user wins + assert result["algorithm:param_b"] == 20.0 # _defaults wins over ini + assert result["algorithm:param_c"] == 55.0 # user-only key present + assert result["algorithm:param_d"] == 3.0 # ini-only key present + + def test_no_ini_params(self, temp_workflow_dir): + """Works when ini_params is None.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}}, + "SomeTool": {"algorithm:other": 7.0} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool") + + assert result["algorithm:param"] == 42.0 + assert result["algorithm:other"] == 7.0 + + def test_different_instances_same_tool(self, temp_workflow_dir): + """Different instance names get independent _defaults and user overrides.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": { + "IDFilter_step1": {"score:min": 0.05}, + "IDFilter_step2": {"score:min": 0.01}, + }, + "IDFilter_step1": {"score:min": 0.001}, + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result1 = pm.get_merged_params("IDFilter_step1", ini_params={"score:min": 0.5}) + result2 = pm.get_merged_params("IDFilter_step2", ini_params={"score:min": 0.5}) + + assert result1["score:min"] == 0.001 # user override for step1 + assert result2["score:min"] == 0.01 # _defaults for step2 (no user override) + + def test_empty_params_json(self, temp_workflow_dir): + """Returns empty dict when params.json is empty and no ini_params.""" + pm = ParameterManager(temp_workflow_dir) + with open(pm.params_file, "w") as f: + json.dump({}, f) + + result = pm.get_merged_params("SomeTool") + + assert result == {} + + +class TestGetToppParametersWithDefaults: + + def test_get_topp_parameters_includes_defaults(self, temp_workflow_dir): + """get_topp_parameters merges _defaults between ini and user values.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}}, + "SomeTool": {"algorithm:other": 99.0} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool", ini_params={"algorithm:param": 1.0}) + assert result["algorithm:param"] == 42.0 + assert result["algorithm:other"] == 99.0 + + +class TestDefaultsSeeding: + + def test_seed_writes_defaults_to_params_json(self, temp_workflow_dir): + """Seeding creates _defaults entry in params.json.""" + pm = ParameterManager(temp_workflow_dir) + custom_defaults = {"param_a": 10.0, "param_b": "fast"} + + # Simulate what input_TOPP seeding does + params = pm.get_parameters_from_json() + if "_defaults" not in params: + params["_defaults"] = {} + params["_defaults"]["MyTool"] = custom_defaults + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Verify + loaded = pm.get_parameters_from_json() + assert loaded["_defaults"]["MyTool"] == {"param_a": 10.0, "param_b": "fast"} + + def test_seed_is_idempotent(self, temp_workflow_dir): + """Seeding the same tool twice overwrites cleanly.""" + pm = ParameterManager(temp_workflow_dir) + + # First seed + params = {"_defaults": {"Tool": {"p1": 1.0}}, "other_key": "keep"} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Second seed with updated defaults + params = pm.get_parameters_from_json() + params["_defaults"]["Tool"] = {"p1": 2.0} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + loaded = pm.get_parameters_from_json() + assert loaded["_defaults"]["Tool"]["p1"] == 2.0 + assert loaded["other_key"] == "keep" + + def test_seed_multiple_instances(self, temp_workflow_dir): + """Different instances of the same tool get independent _defaults.""" + pm = ParameterManager(temp_workflow_dir) + params = { + "_defaults": { + "IDFilter_strict": {"score:pep": 0.01}, + "IDFilter_lenient": {"score:pep": 0.05}, + } + } + with open(pm.params_file, "w") as f: + json.dump(params, f) + + loaded = pm.get_parameters_from_json() + assert loaded["_defaults"]["IDFilter_strict"]["score:pep"] == 0.01 + assert loaded["_defaults"]["IDFilter_lenient"]["score:pep"] == 0.05 + + +try: + import pyopenms as poms + HAS_PYOPENMS = True +except ImportError: + HAS_PYOPENMS = False + + +@pytest.mark.skipif(not HAS_PYOPENMS, reason="pyopenms not available") +class TestSaveParametersWithDefaults: + + def _create_fake_ini(self, pm, tool_name, params_dict): + """Create a fake .ini file with given parameters.""" + param = poms.Param() + for key, value in params_dict.items(): + param.setValue(f"{tool_name}:1:{key}".encode(), value) + poms.ParamXMLFile().store(str(Path(pm.ini_dir, f"{tool_name}.ini")), param) + + def test_value_matching_custom_default_not_saved(self, temp_workflow_dir): + """A value equal to the _defaults entry should not be saved as a user override.""" + pm = ParameterManager(temp_workflow_dir) + + # Create a fake ini with a default value + self._create_fake_ini(pm, "Tool", {"param_a": 10.0}) + + # Pre-seed _defaults with a different value than ini + params = {"_defaults": {"Tool": {"param_a": 42.0}}} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Session state has value matching the custom default (42.0), not the ini default (10.0) + mock_streamlit.session_state[f"{pm.topp_param_prefix}Tool:1:param_a"] = 42.0 + mock_streamlit.session_state["_topp_tool_instance_map"] = {"Tool": "Tool"} + + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # param_a should NOT appear under Tool (it matches the _defaults value) + assert "param_a" not in saved.get("Tool", {}) + # _defaults should still be present + assert saved["_defaults"]["Tool"]["param_a"] == 42.0 + + def test_value_different_from_custom_default_saved(self, temp_workflow_dir): + """A value different from _defaults entry should be saved as user override.""" + pm = ParameterManager(temp_workflow_dir) + + # Create a fake ini with a default value + self._create_fake_ini(pm, "Tool", {"param_a": 10.0}) + + params = {"_defaults": {"Tool": {"param_a": 42.0}}} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + mock_streamlit.session_state[f"{pm.topp_param_prefix}Tool:1:param_a"] = 99.0 + mock_streamlit.session_state["_topp_tool_instance_map"] = {"Tool": "Tool"} + + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + assert saved["Tool"]["param_a"] == 99.0 + + +class TestNonDefaultParamsSummaryDefaults: + + def test_defaults_key_excluded_from_classification(self): + """_defaults dict should not appear as a TOPP tool in the summary.""" + params = { + "_defaults": {"Tool": {"p1": 10}}, + "Tool": {"p1": 20}, + "general_param": "value" + } + # Simulate the classification logic + topp = {} + general = {} + for k, v in params.items(): + if k == "_defaults": + continue + if isinstance(v, dict): + topp[k] = v + else: + general[k] = v + + assert "_defaults" not in topp + assert "Tool" in topp + assert "general_param" in general + + def test_defaults_merged_into_summary(self): + """_defaults values should appear in summary merged with user overrides.""" + params = { + "_defaults": { + "ToolA": {"p1": 10, "p2": 20}, + "ToolB": {"p3": 30} + }, + "ToolA": {"p1": 99} + } + # Simulate the merge logic for summary + topp = {} + for k, v in params.items(): + if k == "_defaults": + continue + if isinstance(v, dict): + topp[k] = v + + defaults = params.get("_defaults", {}) + for tool_name, default_vals in defaults.items(): + if tool_name not in topp: + topp[tool_name] = {} + topp[tool_name] = {**default_vals, **topp.get(tool_name, {})} + + assert topp["ToolA"] == {"p1": 99, "p2": 20} # user override wins for p1 + assert topp["ToolB"] == {"p3": 30} # defaults-only tool appears diff --git a/tests/test_queue_manager_cancel.py b/tests/test_queue_manager_cancel.py index 0f87708..c8ef44a 100644 --- a/tests/test_queue_manager_cancel.py +++ b/tests/test_queue_manager_cancel.py @@ -153,9 +153,7 @@ def test_stopped_status_is_mapped_in_get_job_info(monkeypatch): info = qm.get_job_info("stopped-job") assert info is not None - assert info.status == __import__( - "src.workflow.QueueManager", fromlist=["JobStatus"] - ).JobStatus.CANCELED, ( + assert info.status.name == "CANCELED", ( "RQ 'stopped' status should be reported as CANCELED to the UI; " "otherwise stopped jobs appear stuck in 'queued'." ) diff --git a/tests/test_run_subprocess.py b/tests/test_run_subprocess.py new file mode 100644 index 0000000..cd6889a --- /dev/null +++ b/tests/test_run_subprocess.py @@ -0,0 +1,37 @@ +import pytest +import time +from streamlit.testing.v1 import AppTest + +@pytest.fixture +def launch(): + """Launch the Run Subprocess Streamlit page for testing.""" + + app = AppTest.from_file("content/run_subprocess.py") + app.run(timeout=10) + return app + +def test_file_selection(launch): + """Ensure a file can be selected from the dropdown.""" + launch.run() + + assert len(launch.selectbox) > 0, "No file selection dropdown found!" + + if len(launch.selectbox[0].options) > 0: + launch.selectbox[0].select(launch.selectbox[0].options[0]) + launch.run() + + +def test_extract_ids_button(launch): + """Ensure clicking 'Extract IDs' triggers process and UI updates accordingly.""" + launch.run(timeout=10) + time.sleep(3) + + # Ensure 'Extract ids' button exists + extract_button = next((btn for btn in launch.button if "Extract ids" in btn.label), None) + assert extract_button is not None, "Extract ids button not found!" + + # Click the 'Extract ids' button + extract_button.click() + launch.run(timeout=10) + + print("Extract ids button was clicked successfully!") \ No newline at end of file diff --git a/tests/test_simple_workflow.py b/tests/test_simple_workflow.py new file mode 100644 index 0000000..5a94c41 --- /dev/null +++ b/tests/test_simple_workflow.py @@ -0,0 +1,69 @@ +import pytest +import time +from streamlit.testing.v1 import AppTest + +""" +Tests for the Simple Workflow page functionality. + +These tests verify: +- Number input widgets function correctly +- Session state updates properly +- Table generation with correct dimensions +- Download button presence +""" + +@pytest.fixture +def launch(): + """Launch the Simple Workflow page for testing.""" + app = AppTest.from_file("content/simple_workflow.py") + app.run(timeout=15) + return app + +def test_number_inputs(launch): + """Ensure x and y dimension inputs exist and update correctly.""" + + assert len(launch.number_input) >= 2, f"Expected at least 2 number inputs, found {len(launch.number_input)}" + + # Set x and y dimensions + x_input = next((ni for ni in launch.number_input if ni.key == "example-x-dimension"), None) + y_input = next((ni for ni in launch.number_input if ni.key == "example-y-dimension"), None) + + assert x_input is not None, "X-dimension input not found!" + assert y_input is not None, "Y-dimension input not found!" + + x_input.set_value(5) + y_input.set_value(4) + launch.run(timeout=10) + + # Validate session state updates + assert "example-x-dimension" in launch.session_state, "X-dimension key missing in session state!" + assert "example-y-dimension" in launch.session_state, "Y-dimension key missing in session state!" + assert launch.session_state["example-x-dimension"] == 5, "X-dimension not updated!" + assert launch.session_state["example-y-dimension"] == 4, "Y-dimension not updated!" + + assert len(launch.dataframe) > 0, "Table not generated!" + + df = launch.dataframe[0].value + assert df.shape == (5, 4), f"Expected table size (5,4) but got {df.shape}" + +def test_download_button(launch): + """Ensure 'Download Table' button appears after table generation.""" + + # Locate number inputs by key + x_input = next((ni for ni in launch.number_input if ni.key == "example-x-dimension"), None) + y_input = next((ni for ni in launch.number_input if ni.key == "example-y-dimension"), None) + + assert x_input is not None, "X-dimension input not found!" + assert y_input is not None, "Y-dimension input not found!" + + # Set values and trigger app update + x_input.set_value(3) + y_input.set_value(2) + launch.run(timeout=15) + time.sleep(5) + + assert len(launch.dataframe) > 0, "Table not generated!" + + # Find the "Download Table" button correctly + download_elements = [comp for comp in launch.main if hasattr(comp, "label") and "Download" in comp.label] + assert len(download_elements) > 0, "Download Table button is missing!" diff --git a/tests/test_tool_instance_name.py b/tests/test_tool_instance_name.py new file mode 100644 index 0000000..cd060ca --- /dev/null +++ b/tests/test_tool_instance_name.py @@ -0,0 +1,268 @@ +""" +Tests for the tool_instance_name functionality. + +This module verifies that save_parameters correctly resolves tool instance names +to real tool names when calling create_ini, and that parameters are stored and +retrieved using the instance name as the key. +""" +import os +import sys +import json +import pytest +import tempfile +from pathlib import Path +from unittest.mock import patch, MagicMock, call + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# Mock streamlit before importing ParameterManager +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +_original_streamlit = sys.modules.get('streamlit') +sys.modules['streamlit'] = mock_streamlit + +from src.workflow.ParameterManager import ParameterManager + +if _original_streamlit is not None: + sys.modules['streamlit'] = _original_streamlit +else: + sys.modules.pop('streamlit', None) + +# Remove cached src.workflow modules +for _key in list(sys.modules.keys()): + if _key.startswith('src.workflow'): + sys.modules.pop(_key, None) + + +@pytest.fixture +def temp_workflow_dir(): + """Create a temporary workflow directory for testing.""" + with tempfile.TemporaryDirectory() as tmpdir: + workflow_dir = Path(tmpdir) / "test-workflow" + workflow_dir.mkdir() + ini_dir = workflow_dir / "ini" + ini_dir.mkdir() + yield workflow_dir + + +@pytest.fixture(autouse=True) +def reset_streamlit_state(): + """Reset mock streamlit session state before each test.""" + mock_streamlit.session_state.clear() + yield + + +class TestSaveParametersWithInstanceName: + """Tests for save_parameters correctly resolving tool instance names.""" + + def test_save_parameters_uses_real_tool_name_for_create_ini(self, temp_workflow_dir): + """Test that save_parameters resolves instance name to real tool name + before calling create_ini.""" + pm = ParameterManager(temp_workflow_dir) + + # Simulate session state with an instance name (IDFilter_step1) + # that differs from the real tool name (IDFilter) + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + # Register instance mapping (as input_TOPP would) + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + + # Mock create_ini to track calls - return False (tool not found) + # to prevent further processing that requires actual ini files + with patch.object(pm, 'create_ini', return_value=False) as mock_create_ini: + pm.save_parameters() + + # Verify create_ini was called with the REAL tool name, not the instance name + mock_create_ini.assert_called_once_with("IDFilter") + + def test_save_parameters_without_instance_map_uses_tool_name_directly(self, temp_workflow_dir): + """Test that save_parameters works normally when no instance map exists + (backward compatibility).""" + pm = ParameterManager(temp_workflow_dir) + + # Simulate session state with a normal tool name (no instance mapping) + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter:1:score:pep"] = 0.05 + + with patch.object(pm, 'create_ini', return_value=False) as mock_create_ini: + pm.save_parameters() + + # Should use the tool name directly + mock_create_ini.assert_called_once_with("IDFilter") + + def test_save_parameters_stores_under_instance_name(self, temp_workflow_dir): + """Test that parameters are stored in JSON under the instance name, not + the real tool name.""" + pm = ParameterManager(temp_workflow_dir) + + # Create a mock ini file for IDFilter + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + # Set up instance mapping + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + + # Mock pyopenms Param and ParamXMLFile to avoid needing real ini files + mock_param = MagicMock() + mock_param.getValue.return_value = 0.01 # Different from session state value + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile') as mock_xml: + pm.save_parameters() + + # Load saved parameters + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # Parameters should be stored under the instance name + assert "IDFilter_step1" in saved + assert saved["IDFilter_step1"]["score:pep"] == 0.05 + + def test_save_parameters_multiple_instances_same_tool(self, temp_workflow_dir): + """Test that two instances of the same tool get separate parameter entries.""" + pm = ParameterManager(temp_workflow_dir) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + # Set up two instances with different parameter values + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter", + "IDFilter_step2": "IDFilter", + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.01 + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step2:1:score:pep"] = 0.05 + + mock_param = MagicMock() + mock_param.getValue.return_value = 0.0 # Default differs from both + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # Both instances should have separate entries + assert "IDFilter_step1" in saved + assert "IDFilter_step2" in saved + assert saved["IDFilter_step1"]["score:pep"] == 0.01 + assert saved["IDFilter_step2"]["score:pep"] == 0.05 + + def test_save_parameters_ini_key_maps_instance_to_real_tool(self, temp_workflow_dir): + """Test that ini_key correctly maps instance name back to real tool name + for param.getValue lookup.""" + pm = ParameterManager(temp_workflow_dir) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + + mock_param = MagicMock() + mock_param.getValue.return_value = 0.01 + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + pm.save_parameters() + + # Verify that param.getValue was called with the REAL tool name key + # (IDFilter:1:score:pep), not the instance name key (IDFilter_step1:1:score:pep) + mock_param.getValue.assert_called_with(b"IDFilter:1:score:pep") + + def test_save_parameters_display_keys_skipped_with_instance_name(self, temp_workflow_dir): + """Test that _display keys are still skipped when using instance names.""" + pm = ParameterManager(temp_workflow_dir) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep_display"] = ["0.05"] + + mock_param = MagicMock() + mock_param.getValue.return_value = 0.01 + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # _display key should not be stored + assert "score:pep_display" not in saved.get("IDFilter_step1", {}) + assert "score:pep" in saved.get("IDFilter_step1", {}) + + +class TestGetToppParametersWithInstanceName: + """Tests for get_topp_parameters with tool_instance_name.""" + + def test_get_topp_parameters_with_instance_name(self, temp_workflow_dir): + """Test that get_topp_parameters uses instance name for JSON lookup.""" + pm = ParameterManager(temp_workflow_dir) + + # Create params.json with instance-keyed parameters + params = { + "IDFilter_step1": { + "score:pep": 0.05 + } + } + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Create a mock ini file + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_param = MagicMock() + mock_param.keys.return_value = [b"IDFilter:1:score:pep"] + mock_param.getValue.return_value = 0.01 # default + + with patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + result = pm.get_topp_parameters("IDFilter", tool_instance_name="IDFilter_step1") + + # Should return the instance-specific value + assert result["score:pep"] == 0.05 + + def test_get_topp_parameters_without_instance_name_backward_compat(self, temp_workflow_dir): + """Test that get_topp_parameters works without instance name (backward compat).""" + pm = ParameterManager(temp_workflow_dir) + + params = { + "IDFilter": { + "score:pep": 0.05 + } + } + with open(pm.params_file, "w") as f: + json.dump(params, f) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_param = MagicMock() + mock_param.keys.return_value = [b"IDFilter:1:score:pep"] + mock_param.getValue.return_value = 0.01 + + with patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + result = pm.get_topp_parameters("IDFilter") + + assert result["score:pep"] == 0.05 diff --git a/tests/test_topp_flag_parameters.py b/tests/test_topp_flag_parameters.py new file mode 100644 index 0000000..627f50e --- /dev/null +++ b/tests/test_topp_flag_parameters.py @@ -0,0 +1,346 @@ +""" +Unit tests for PR #397 — flag parameter support for TOPP tools. + +PR #397 lets a caller mark certain TOPP parameters as CLI *flags*: parameters +passed by presence only (e.g. ``-force``), without a trailing value. The flag +names are persisted per tool instance by ``input_TOPP()`` into both +``st.session_state["_topp_flag_params"]`` and ``params.json["_flag_params"]``, +and consumed by ``run_topp()`` in ``src/workflow/CommandExecutor.py`` when it +builds the command line. + +These tests exercise ``run_topp()`` — the consumer that turns the persisted flag +definitions and merged parameters into an actual command. Driving ``run_topp()`` +also validates the persistence *contract* (the exact ``_flag_params`` / +``_topp_flag_params`` shapes that ``input_TOPP()`` writes), which is where the two +halves of the feature meet. + +The suite covers the working behaviour of the feature and also guards the two +issues CodeRabbit flagged during review, which are now fixed in ``run_topp()``: + - Finding 1 (per-tool flag fallback): + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023551 + - Finding 2 (list expansion / empty-list skipping): + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 +The tests that pin those findings carry a "CodeRabbit finding N" note in their +docstrings and live alongside the related behaviour they protect. +""" +import os +import sys +import json +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# --------------------------------------------------------------------------- +# Import the modules under test with `streamlit` (and `pyopenms`) mocked at the +# sys.modules level. Both CommandExecutor and ParameterManager do +# `import streamlit as st` at module top (ParameterManager also +# `import pyopenms as poms`). run_topp() only needs `st.session_state` to behave +# like a plain dict and never touches pyopenms, so lightweight mocks keep the +# test runnable without those heavy deps installed while still exercising the +# real command-construction logic. Mirrors tests/test_tool_instance_name.py. +# --------------------------------------------------------------------------- +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +_original_streamlit = sys.modules.get("streamlit") +_original_pyopenms = sys.modules.get("pyopenms") +sys.modules["streamlit"] = mock_streamlit +if _original_pyopenms is None: + sys.modules["pyopenms"] = MagicMock() + +from src.workflow.ParameterManager import ParameterManager +from src.workflow.CommandExecutor import CommandExecutor + +# Restore the original modules so other test files import the real ones. The +# classes imported above keep their module-level `st`/`poms` bound to the mocks. +if _original_streamlit is not None: + sys.modules["streamlit"] = _original_streamlit +else: + sys.modules.pop("streamlit", None) +if _original_pyopenms is None: + sys.modules.pop("pyopenms", None) + +for _key in list(sys.modules.keys()): + if _key.startswith("src.workflow"): + sys.modules.pop(_key, None) + + +TOOL = "FeatureFinderMetabo" + + +@pytest.fixture(autouse=True) +def reset_session_state(): + """Give each test a fresh, empty mocked session_state.""" + mock_streamlit.session_state = {} + yield + mock_streamlit.session_state = {} + + +def build_command( + params_json=None, + session_state=None, + *, + tool=TOOL, + input_output=None, + custom_params=None, + tool_instance_name=None, +): + """ + Invoke ``run_topp()`` with the supplied ``params.json`` content and + ``session_state``, and return the single command list it builds. + + ``run_command`` / ``run_multiple_commands`` are stubbed so nothing is + executed; the built command is captured from the ``run_command`` mock. + ``max_threads`` is pinned to 1 so the trailing ``-threads`` argument is + deterministic. + """ + if input_output is None: + input_output = {"in": ["input.mzML"], "out": ["output.featureXML"]} + + params_json = dict(params_json or {}) + params_json.setdefault("max_threads", 1) + + with tempfile.TemporaryDirectory() as tmpdir: + workflow_dir = Path(tmpdir) + pm = ParameterManager(workflow_dir) + with open(pm.params_file, "w", encoding="utf-8") as f: + json.dump(params_json, f) + + mock_streamlit.session_state = dict(session_state or {}) + + executor = CommandExecutor(workflow_dir, MagicMock(), pm) + executor.run_command = MagicMock(return_value=True) + executor.run_multiple_commands = MagicMock(return_value=True) + + executor.run_topp( + tool, + input_output, + custom_params=custom_params or {}, + tool_instance_name=tool_instance_name or tool, + ) + + assert executor.run_command.call_count == 1, ( + "expected exactly one single-process command, got " + f"{executor.run_command.call_count}" + ) + return executor.run_command.call_args.args[0] + + +# --------------------------- assertion helpers ----------------------------- + +def has_flag(cmd, name): + """True if ``-name`` appears anywhere in the command.""" + return f"-{name}" in cmd + + +def token_after(cmd, name): + """The single token immediately following ``-name`` (or None if it is last).""" + idx = cmd.index(f"-{name}") + return cmd[idx + 1] if idx + 1 < len(cmd) else None + + +def values_after(cmd, name): + """All value tokens following ``-name`` up to the next ``-flag`` token.""" + idx = cmd.index(f"-{name}") + vals = [] + for tok in cmd[idx + 1:]: + if tok.startswith("-"): + break + vals.append(tok) + return vals + + +def is_bare_flag(cmd, name): + """True if ``-name`` is present with no value (next token is another flag).""" + if not has_flag(cmd, name): + return False + nxt = token_after(cmd, name) + return nxt is None or nxt.startswith("-") + + +# ============================ working behaviour ============================ + + +class TestCommandSkeleton: + def test_input_output_files_prefixed(self): + cmd = build_command() + assert cmd[0] == TOOL + assert cmd[1:5] == ["-in", "input.mzML", "-out", "output.featureXML"] + # threads pinned to 1 and always appended last + assert cmd[-2:] == ["-threads", "1"] + + def test_collected_files_passed_as_single_list(self): + # A [["a", "b"]] entry is expanded in place after its -key. + cmd = build_command(input_output={"in": [["a.mzML", "b.mzML"]], "out": ["c.featureXML"]}) + assert cmd[1:4] == ["-in", "a.mzML", "b.mzML"] + + +class TestFlagParameters: + """Flags emit a bare ``-key`` when enabled and nothing when disabled.""" + + def test_flag_true_bool_emits_bare_flag(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": True}} + ) + assert is_bare_flag(cmd, "force") + assert "True" not in cmd + + def test_flag_string_true_emits_bare_flag(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": "true"}} + ) + assert is_bare_flag(cmd, "force") + assert "true" not in cmd + + def test_flag_string_true_is_case_insensitive(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": "True"}} + ) + assert is_bare_flag(cmd, "force") + + def test_flag_false_bool_omitted(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": False}} + ) + assert not has_flag(cmd, "force") + + def test_flag_string_false_omitted(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": "false"}} + ) + assert not has_flag(cmd, "force") + + +class TestRegularParameters: + """Non-flag merged parameters keep the existing value-appending behaviour.""" + + def test_empty_string_skipped(self): + cmd = build_command({TOOL: {"opt": ""}}) + assert not has_flag(cmd, "opt") + + def test_none_skipped(self): + cmd = build_command({TOOL: {"opt": None}}) + assert not has_flag(cmd, "opt") + + def test_zero_is_preserved(self): + # 0 and 0.0 are valid values, not "empty" — they must be passed through. + cmd = build_command({TOOL: {"min_int": 0, "min_float": 0.0}}) + assert values_after(cmd, "min_int") == ["0"] + assert values_after(cmd, "min_float") == ["0.0"] + + def test_scalar_value_appended(self): + cmd = build_command({TOOL: {"mz_tolerance": 10.5}}) + assert values_after(cmd, "mz_tolerance") == ["10.5"] + + def test_multiline_string_split_into_args(self): + cmd = build_command({TOOL: {"seq": "ALPHA\nBETA\nGAMMA"}}) + assert values_after(cmd, "seq") == ["ALPHA", "BETA", "GAMMA"] + + def test_merged_list_param_expanded(self): + """ + CodeRabbit finding 2 (fixed): a list-valued merged parameter expands into + separate CLI args, not its Python ``str()`` (e.g. ``"['a', 'b']"``). + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 + """ + cmd = build_command({TOOL: {"ids": ["a", "b"]}}) + assert values_after(cmd, "ids") == ["a", "b"] + + def test_merged_empty_list_param_skipped(self): + """ + CodeRabbit finding 2 (fixed): an empty-list merged parameter is omitted + entirely rather than emitting a ``-key`` with no usable value. + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 + """ + cmd = build_command({TOOL: {"ids": []}}) + assert not has_flag(cmd, "ids") + + +class TestCustomParameters: + """custom_params share the flag set and expand non-empty lists.""" + + def test_custom_flag_truthy_bare(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}}, + custom_params={"force": True}, + ) + assert is_bare_flag(cmd, "force") + + def test_custom_flag_false_omitted(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}}, + custom_params={"force": False}, + ) + assert not has_flag(cmd, "force") + + def test_custom_scalar_value(self): + cmd = build_command(custom_params={"extra": 5}) + assert values_after(cmd, "extra") == ["5"] + + def test_custom_nonempty_list_expanded(self): + cmd = build_command(custom_params={"ids": ["a", "b", "c"]}) + assert values_after(cmd, "ids") == ["a", "b", "c"] + + def test_custom_empty_string_skipped(self): + cmd = build_command(custom_params={"opt": ""}) + assert not has_flag(cmd, "opt") + + def test_custom_empty_list_param_skipped(self): + """ + CodeRabbit finding 2 (fixed): an empty-list custom parameter is omitted + rather than emitting a bare ``-key`` with no value. + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 + """ + cmd = build_command(custom_params={"ids": []}) + assert not has_flag(cmd, "ids") + + +class TestFlagSourceContract: + """Where run_topp() reads the flag definitions from.""" + + def test_flag_params_loaded_from_params_json(self): + # Survives a session restart: only params.json carries the flag list. + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": True}}, + session_state={}, + ) + assert is_bare_flag(cmd, "force") + + def test_fallback_to_session_state_when_json_has_no_flags(self): + # params.json has no _flag_params at all -> live session_state is used. + cmd = build_command( + {TOOL: {"force": True}}, + session_state={"_topp_flag_params": {TOOL: ["force"]}}, + ) + assert is_bare_flag(cmd, "force") + + def test_params_json_takes_priority_over_session_state(self): + # params.json says "force" is a flag; session_state disagrees (empty). + # params.json wins, so force is treated as a flag (bare, no value). + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": True}}, + session_state={"_topp_flag_params": {TOOL: []}}, + ) + assert is_bare_flag(cmd, "force") + assert "True" not in cmd + + def test_flag_fallback_uses_current_tool_when_other_tool_has_flags(self): + """ + CodeRabbit finding 1 (fixed): when params.json._flag_params holds an entry + for a DIFFERENT tool, the current tool's flags must still be read from the + session_state fallback. Previously the global ``if not flag_map`` check + skipped the fallback whenever any tool had flags, so the current tool's + flag was treated as a regular parameter and emitted as ``-force True``. + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023551 + """ + cmd = build_command( + {"_flag_params": {"OtherTool": ["some_flag"]}, TOOL: {"force": True}}, + session_state={"_topp_flag_params": {TOOL: ["force"]}}, + ) + assert is_bare_flag(cmd, "force") + assert "True" not in cmd