From 3016b0a2c24599da1a9a5e9cde827075d0431a30 Mon Sep 17 00:00:00 2001 From: jkvision1101 Date: Thu, 16 Jul 2026 10:34:59 +0900 Subject: [PATCH 01/10] Revert "feat: integrate LFQ and TMT workflows (#27)" This reverts commit 83646ecb68387b2da2f0bb57c8f6aa1c693dbe15. --- app.py | 2 +- content/results_abundance.py | 135 +- content/results_heatmap.py | 130 +- content/results_pathway_analysis.py | 258 ---- content/results_pca.py | 45 +- content/results_volcano.py | 240 ++-- src/WorkflowTest.py | 1829 +++++++++------------------ src/common/results_helpers.py | 351 ++--- src/workflow/CommandExecutor.py | 81 +- src/workflow/ParameterManager.py | 24 - src/workflow/StreamlitUI.py | 126 +- 11 files changed, 915 insertions(+), 2306 deletions(-) delete mode 100644 content/results_pathway_analysis.py diff --git a/app.py b/app.py index 97f9a16..194d857 100644 --- a/app.py +++ b/app.py @@ -27,7 +27,7 @@ st.Page(Path("content", "results_pca.py"), title="PCA", icon="πŸ“Š"), st.Page(Path("content", "results_heatmap.py"), title="Heatmap", icon="πŸ”₯"), st.Page(Path("content", "results_library.py"), title="Spectral Library", icon="πŸ“š"), - st.Page(Path("content", "results_pathway_analysis.py"), title="Pathway Analysis", icon="πŸ“‰"), + st.Page(Path("content", "results_proteomicslfq.py"), title="Proteomics LFQ", icon="πŸ§ͺ"), ], } diff --git a/content/results_abundance.py b/content/results_abundance.py index a86f1a3..a7ff453 100644 --- a/content/results_abundance.py +++ b/content/results_abundance.py @@ -4,7 +4,6 @@ from pathlib import Path from src.common.common import page_setup from src.common.results_helpers import get_workflow_dir, get_abundance_data -from src.workflow.ParameterManager import ParameterManager params = page_setup() st.title("Abundance Quantification") @@ -22,10 +21,6 @@ workflow_dir = get_workflow_dir(st.session_state["workspace"]) quant_dir = workflow_dir / "results" / "quant_results" -parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - -workflow_params = parameter_manager.get_parameters_from_json() -analysis_mode = workflow_params.get("analysis-mode", "LFQ") if not quant_dir.exists(): st.info("No quantification results available yet. Please run the workflow first.") @@ -40,60 +35,6 @@ csv_file = csv_files[0] -def render_protein_table(pivot_df, group_map, is_lfq=True): - """Common function to render the protein-level abundance table""" - st.markdown("### Protein-Level Abundance Table") - st.info( - "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." - ) - - # Display group comparison info - groups = sorted(set(group_map.values())) - if len(groups) >= 2: - group1, group2 = sorted(groups)[:2] - st.info(f"Statistical comparison: **{group2} vs {group1}**") - - if is_lfq: - # Handle LFQ mode columns (Raw Intensity) - id_col = "ProteinName" - exclude_cols = [id_col, "log2FC", "p-value", "PeptideSequence"] - sample_cols = [c for c in pivot_df.columns if c not in exclude_cols] - - pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) - display_cols = [id_col, "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"] - help_text = "Raw sample intensities" - y_min = None - else: - # Handle non-LFQ mode columns (Log2-transformed Intensity) - id_col = "protein" - exclude_cols = [id_col, "log2FC", "p-value", "p-adj", "n_proteins", "n_peptides", "protein_score"] - sample_cols = [c for c in pivot_df.columns if c not in exclude_cols and "ratio" not in c.lower()] - - pivot_df["Intensity"] = pivot_df[sample_cols].apply( - lambda row: [np.log2(v + 1) for v in row], axis=1 - ) - display_cols = [id_col, "log2FC", "p-value", "Intensity"] + sample_cols - help_text = "Sample intensities (log2 scale)" - y_min = 0 - - # Filter to available columns, then sort and display - available_cols = [c for c in display_cols if c in pivot_df.columns] - - st.dataframe( - pivot_df[available_cols].sort_values("p-value"), - column_config={ - "Intensity": st.column_config.BarChartColumn( - "Intensity", - help=help_text, - width="small", - y_min=y_min, - ), - }, - use_container_width=True, - ) - protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) try: @@ -103,44 +44,58 @@ def render_protein_table(pivot_df, group_map, is_lfq=True): st.info("No data found in this file.") st.stop() - result = get_abundance_data(st.session_state["workspace"]) + with protein_tab: + st.markdown("### Protein-Level Abundance Table") - if analysis_mode == "LFQ": - protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) - - with protein_tab: - if result is None: - st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.") - # st.page_link("content/workflow_configure.py", label="Go to Configure", icon="βš™οΈ") - st.stop() - - pivot_df, expr_df, group_map = result - render_protein_table(pivot_df, group_map, is_lfq=True) - - with psm_tab: - st.markdown("### PSM-level Quantification Table") - st.info( - "This table shows the PSM-level quantification data, including protein IDs, " - "peptide sequences, charge states, and intensities across samples. " - "Each row represents one peptide-spectrum match detected from the MS/MS analysis." - ) - st.dataframe(df, use_container_width=True) - - else: - pre_processing_tab, protein_tab = st.tabs(["Pre-processing", "Protein Table"]) + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples.\n\n" + "Additionally, log2 fold change and p-values are calculated between sample groups." + ) + result = get_abundance_data(st.session_state["workspace"]) if result is None: - st.info("πŸ’‘ Please complete the configuration in the 'Configure' page to see results.") + st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.") + st.page_link("content/workflow_configure.py", label="Go to Configure", icon="βš™οΈ") st.stop() - + pivot_df, expr_df, group_map = result - with pre_processing_tab: - st.write("### Final Results (Group row removed, Stats added)") - st.dataframe(pivot_df.head(10)) + # Display group comparison info + groups = sorted(set(group_map.values())) + if len(groups) >= 2: + group1, group2 = sorted(groups)[:2] + st.info(f"Statistical comparison: **{group2} vs {group1}**") + + # Get sample columns (between stats and PeptideSequence) + sample_cols = [c for c in pivot_df.columns if c not in ["ProteinName", "log2FC", "p-value", "PeptideSequence"]] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) - with protein_tab: - render_protein_table(pivot_df, group_map, is_lfq=False) + # Reorder columns: place Intensity after p-value + display_cols = ["ProteinName", "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"] + display_df = pivot_df[display_cols] + + st.dataframe( + display_df.sort_values("p-value"), + column_config={ + "Intensity": st.column_config.BarChartColumn( + "Intensity", + help="Raw sample intensities", + width="small", + ), + }, + use_container_width=True, + ) + + with psm_tab: + st.markdown("### PSM-level Quantification Table") + st.info( + "This table shows the PSM-level quantification data, including protein IDs, " + "peptide sequences, charge states, and intensities across samples. " + "Each row represents one peptide-spectrum match detected from the MS/MS analysis." + ) + st.dataframe(df, use_container_width=True) except Exception as e: st.error(f"Failed to load {csv_file.name}: {e}") diff --git a/content/results_heatmap.py b/content/results_heatmap.py index 72b8438..4ece3f4 100644 --- a/content/results_heatmap.py +++ b/content/results_heatmap.py @@ -5,8 +5,7 @@ from scipy.cluster.hierarchy import linkage, leaves_list from scipy.spatial.distance import pdist from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data, get_workflow_dir -from src.workflow.ParameterManager import ParameterManager +from src.common.results_helpers import get_abundance_data params = page_setup() st.title("Heatmap") @@ -30,103 +29,48 @@ pivot_df, expr_df, group_map = result -workflow_dir = get_workflow_dir(st.session_state["workspace"]) -parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") +top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") -workflow_params = parameter_manager.get_parameters_from_json() -analysis_mode = workflow_params.get("analysis-mode", "LFQ") +var_series = expr_df.var(axis=1) +top_proteins = var_series.sort_values(ascending=False).head(top_n).index +heatmap_df = expr_df.loc[top_proteins] +heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) +heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() -st.write("Workflow Analysis Mode:", analysis_mode) +if not heatmap_z.empty: + row_linkage = linkage(pdist(heatmap_z.values), method="average") + row_order = leaves_list(row_linkage) -if analysis_mode == "LFQ": - top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") + col_linkage = linkage(pdist(heatmap_z.T.values), method="average") + col_order = leaves_list(col_linkage) - var_series = expr_df.var(axis=1) - top_proteins = var_series.sort_values(ascending=False).head(top_n).index - heatmap_df = expr_df.loc[top_proteins] - heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) - heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() + heatmap_clustered = heatmap_z.iloc[row_order, col_order] - if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) + fig_heatmap = px.imshow( + heatmap_clustered, + labels=dict(x="Sample", y="Protein", color="Z-score"), + aspect="auto", + color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], + zmin=-3, zmax=3 + ) - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) + fig_heatmap.update_layout( + height=700, + xaxis={'side': 'bottom'}, + yaxis={'side': 'left'} + ) - heatmap_clustered = heatmap_z.iloc[row_order, col_order] + fig_heatmap.update_xaxes(tickfont=dict(size=10)) + fig_heatmap.update_yaxes(tickfont=dict(size=8)) - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) - - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} - ) - - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, use_container_width=True) - else: - st.warning("Insufficient data to generate the heatmap.") - - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") - with col2: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") + st.plotly_chart(fig_heatmap, use_container_width=True) else: - top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") - - var_series = expr_df.var(axis=1) - top_proteins = var_series.sort_values(ascending=False).head(top_n).index - heatmap_df = expr_df.loc[top_proteins] - heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) - heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() - - if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) - - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) - - heatmap_clustered = heatmap_z.iloc[row_order, col_order] - - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) - - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} - ) - - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, width="stretch") - else: - st.warning("Insufficient data to generate the heatmap.") - - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") - with col2: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") + st.warning("Insufficient data to generate the heatmap.") + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") +with col2: + st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") diff --git a/content/results_pathway_analysis.py b/content/results_pathway_analysis.py deleted file mode 100644 index f5eb4c1..0000000 --- a/content/results_pathway_analysis.py +++ /dev/null @@ -1,258 +0,0 @@ -import json -import mygene -import streamlit as st -import pandas as pd -import numpy as np -import plotly.express as px -import plotly.io as pio -from collections import defaultdict -from scipy.stats import fisher_exact -from pathlib import Path -from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data - -# ================================ -# Page setup -# ================================ -params = page_setup() -st.title("ProteomicsLFQ Results") - -# ================================ -# Workspace check -# ================================ -if "workspace" not in st.session_state: - st.warning("Please initialize your workspace first.") - st.stop() - -# ================================ -# _run_go_enrichment function -# ================================ -def _run_go_enrichment(pivot_df: pd.DataFrame, results_dir: Path): - p_cutoff = 0.05 - fc_cutoff = 1.0 - - analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() - - if analysis_df.empty: - st.error("No valid statistical data found for GO enrichment.") - st.write("❗ analysis_df is empty") - else: - with st.spinner("Fetching GO terms from MyGene.info API..."): - mg = mygene.MyGeneInfo() - - def get_clean_uniprot(name): - parts = str(name).split("|") - return parts[1] if len(parts) >= 2 else parts[0] - - analysis_df["UniProt"] = analysis_df["protein"].apply(get_clean_uniprot) - - bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() - fg_ids = analysis_df[ - (analysis_df["p-value"] < p_cutoff) & - (analysis_df["log2FC"].abs() >= fc_cutoff) - ]["UniProt"].dropna().astype(str).unique().tolist() - # st.write("βœ… get_clean_uniprot applied") - - if len(fg_ids) < 3: - st.warning( - f"Not enough significant proteins " - f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " - f"Found: {len(fg_ids)}" - ) - st.write("❗ Not enough significant proteins") - else: - res_list = mg.querymany( - bg_ids, scopes="uniprot", fields="go", as_dataframe=False - ) - res_go = pd.DataFrame(res_list) - if "notfound" in res_go.columns: - res_go = res_go[res_go["notfound"] != True] - - def extract_go_terms(go_data, go_type): - if not isinstance(go_data, dict) or go_type not in go_data: - return [] - terms = go_data[go_type] - if isinstance(terms, dict): - terms = [terms] - return list({t.get("term") for t in terms if "term" in t}) - - for go_type in ["BP", "CC", "MF"]: - res_go[f"{go_type}_terms"] = res_go["go"].apply( - lambda x: extract_go_terms(x, go_type) - ) - - annotated_ids = set(res_go["query"].astype(str)) - fg_set = annotated_ids.intersection(fg_ids) - bg_set = annotated_ids - # st.write(f"βœ… fg_set bg_set are set") - - def run_go(go_type): - go2fg = defaultdict(set) - go2bg = defaultdict(set) - - for _, row in res_go.iterrows(): - uid = str(row["query"]) - for term in row[f"{go_type}_terms"]: - go2bg[term].add(uid) - if uid in fg_set: - go2fg[term].add(uid) - - records = [] - N_fg = len(fg_set) - N_bg = len(bg_set) - - for term, fg_genes in go2fg.items(): - a = len(fg_genes) - if a == 0: - continue - b = N_fg - a - c = len(go2bg[term]) - a - d = N_bg - (a + b + c) - - _, p = fisher_exact([[a, b], [c, d]], alternative="greater") - records.append({ - "GO_Term": term, - "Count": a, - "GeneRatio": f"{a}/{N_fg}", - "p_value": p, - }) - - df = pd.DataFrame(records) - if df.empty: - return None, None - - df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) - df = df.sort_values("p_value").head(20) - - # βœ… Plotly Figure - fig = px.bar( - df, - x="-log10(p)", - y="GO_Term", - orientation="h", - title=f"GO Enrichment ({go_type})", - ) - - # st.write(f"βœ… Plotly Figure generated") - - fig.update_layout( - yaxis=dict(autorange="reversed"), - height=500, - margin=dict(l=10, r=10, t=40, b=10), - ) - - return fig, df - - go_results = {} - - for go_type in ["BP", "CC", "MF"]: - fig, df_go = run_go(go_type) - if fig is not None: - go_results[go_type] = { - "fig": fig, - "df": df_go - } - # st.write(f"βœ… go_type generated") - - go_dir = results_dir / "go-terms" - go_dir.mkdir(parents=True, exist_ok=True) - - go_data = {} - - for go_type in ["BP", "CC", "MF"]: - if go_type in go_results: - fig = go_results[go_type]["fig"] - df = go_results[go_type]["df"] - - go_data[go_type] = { - "fig_json": fig.to_json(), # Figure β†’ JSON string - "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts - } - - go_json_file = go_dir / "go_results.json" - with open(go_json_file, "w") as f: - json.dump(go_data, f) - st.session_state["go_results"] = go_results - st.session_state["go_ready"] = True if go_data else False - # st.write("βœ… GO enrichment analysis complete") - -# ================================ -# Load abundance data -# ================================ -results_dir = Path(st.session_state["workspace"]) / "topp-workflow" / "results" / "quant_results" -result = get_abundance_data(st.session_state["workspace"]) -if result is None: - st.info("Abundance data not available. Please run the workflow and configure sample groups first.") - st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") - st.stop() - -pivot_df, expr_df, group_map = result - -go_json_file = results_dir / "go-terms" / "go_results.json" - -go_input_df = pivot_df.copy() -if "ProteinName" in go_input_df.columns: - go_input_df = go_input_df.rename(columns={"ProteinName": "protein"}) - -_run_go_enrichment(go_input_df, results_dir) - -# ================================ -# Tabs -# ================================ -protein_tab, = st.tabs(["🧬 Protein Table"]) - -# ================================ -# Protein-level results -# ================================ -with protein_tab: - st.markdown("### 🧬 Protein-Level Abundance Table") - st.info( - "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." - ) - - if pivot_df.empty: - st.info("No protein-level data available.") - else: - st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df.sort_values("p-value"), width="stretch") - -# ====================================================== -# GO Enrichment Results -# ====================================================== -st.markdown("---") -st.subheader("🧬 GO Enrichment Analysis") - -if not go_json_file.exists(): - st.info("GO Enrichment results are not available yet. Please run the analysis first.") -else: - with open(go_json_file, "r") as f: - go_data = json.load(f) - - bp_tab, cc_tab, mf_tab = st.tabs([ - "🧬 Biological Process", - "🏠 Cellular Component", - "βš™οΈ Molecular Function", - ]) - - for tab, go_type in zip([bp_tab, cc_tab, mf_tab], ["BP", "CC", "MF"]): - with tab: - if go_type not in go_data: - st.info(f"No enriched {go_type} terms found.") - continue - - fig_json = go_data[go_type]["fig_json"] - df_dict = go_data[go_type]["df_dict"] - - fig = pio.from_json(fig_json) - - df_go = pd.DataFrame(df_dict) - - if df_go.empty: - st.info(f"No enriched {go_type} terms found.") - else: - st.plotly_chart(fig, width="stretch") - - st.markdown(f"#### {go_type} Enrichment Results") - st.dataframe(df_go, width="stretch") \ No newline at end of file diff --git a/content/results_pca.py b/content/results_pca.py index 466e475..45ea8eb 100644 --- a/content/results_pca.py +++ b/content/results_pca.py @@ -5,8 +5,7 @@ from sklearn.decomposition import PCA from sklearn.preprocessing import StandardScaler from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data, get_workflow_dir -from src.workflow.ParameterManager import ParameterManager +from src.common.results_helpers import get_abundance_data params = page_setup() st.title("PCA Analysis") @@ -30,25 +29,13 @@ pivot_df, expr_df, group_map = result -workflow_dir = get_workflow_dir(st.session_state["workspace"]) -parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") -workflow_params = parameter_manager.get_parameters_from_json() -analysis_mode = workflow_params.get("analysis-mode", "LFQ") - -st.write("Workflow Analysis Mode:", analysis_mode) - top_n = 500 -if analysis_mode == "LFQ": - protein_col = "ProteinName" -else: - protein_col = "protein" - top_proteins = ( pivot_df .dropna(subset=["p-adj"]) .sort_values("p-adj", ascending=True) - .head(top_n)[protein_col] + .head(top_n)["ProteinName"] ) expr_df_pca = expr_df.loc[ @@ -71,25 +58,10 @@ index=X.index ) -if analysis_mode == "LFQ": - norm_map = { - k.replace(".mzML", ""): v - for k, v in group_map.items() - } -else: - actual_sample_names = pca_df.index.tolist() - norm_map = {} - for k, v in group_map.items(): - try: - sample_idx = int(k) + 1 - target_substring = f"sample{sample_idx}[" - real_full_name = next((name for name in actual_sample_names if target_substring in name), None) - - if real_full_name: - norm_map[real_full_name] = v if v and v.strip() else "Unassigned" - except ValueError: - continue - +norm_map = { + k.replace(".mzML", ""): v + for k, v in group_map.items() +} pca_df["Group"] = pca_df.index.map(norm_map) fig_pca = px.scatter( @@ -107,9 +79,8 @@ height=600, ) -st.plotly_chart(fig_pca, width="stretch") +st.plotly_chart(fig_pca, use_container_width=True) -st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") st.markdown("---") @@ -118,4 +89,4 @@ with col1: st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") with col2: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") \ No newline at end of file + st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") diff --git a/content/results_volcano.py b/content/results_volcano.py index 5cc9b29..8502489 100644 --- a/content/results_volcano.py +++ b/content/results_volcano.py @@ -3,8 +3,7 @@ import plotly.express as px import numpy as np from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data, get_workflow_dir -from src.workflow.ParameterManager import ParameterManager +from src.common.results_helpers import get_abundance_data params = page_setup() st.title("Volcano Plot") @@ -29,164 +28,79 @@ pivot_df, expr_df, group_map = result if pivot_df.empty: - st.info("No data available for volcano plot.") - st.stop() - -workflow_dir = get_workflow_dir(st.session_state["workspace"]) -parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - -workflow_params = parameter_manager.get_parameters_from_json() -analysis_mode = workflow_params.get("analysis-mode", "LFQ") - -st.write("Workflow Analysis Mode:", analysis_mode) - -if analysis_mode == "LFQ": - volcano_df = pivot_df.copy() - volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"]) - - volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) - - fc_thresh = st.slider( - "log2 Fold Change threshold", - min_value=0.5, - max_value=3.0, - value=1.0, - step=0.1, - ) - - p_thresh = st.slider( - "p-adj (FDR) threshold", - min_value=0.001, - max_value=0.1, - value=0.05, - step=0.001, - ) - - volcano_df["Significance"] = "Not significant" - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", - ] = "Up-regulated" - - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", - ] = "Down-regulated" - - fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["ProteinName", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } - ) - - fig_volcano.add_vline(x=fc_thresh, line_dash="dash") - fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") - fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - - # Make x-axis symmetric around zero - max_abs_fc = volcano_df["log2FC"].abs().max() - x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding - - fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, - height=600, - ) - - st.plotly_chart(fig_volcano, use_container_width=True) - - up_count = (volcano_df["Significance"] == "Up-regulated").sum() - down_count = (volcano_df["Significance"] == "Down-regulated").sum() - st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") - - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") - with col2: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") -else: - # Threshold Selection UI - st.divider() - c1, c2 = st.columns(2) - with c1: - fc_thresh = st.slider( - "log2 Fold Change threshold", - min_value=0.1, - max_value=3.0, - value=1.0, - step=0.1, - ) - with c2: - p_thresh = st.slider( - "p-adj (FDR) threshold", - min_value=0.001, - max_value=0.1, - value=0.05, - step=0.001, - ) - - volcano_df = pivot_df.dropna(subset=["log2FC", "p-adj"]).copy() - volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) - - volcano_df["Significance"] = "Not significant" - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", - ] = "Up-regulated" - - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", - ] = "Down-regulated" - - fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["protein", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } - ) - - fig_volcano.add_vline(x=fc_thresh, line_dash="dash") - fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") - fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - - # Make x-axis symmetric around zero - max_abs_fc = volcano_df["log2FC"].abs().max() - x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding - - fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, - height=600, - ) - - st.plotly_chart(fig_volcano, width="stretch") - - up_count = (volcano_df["Significance"] == "Up-regulated").sum() - down_count = (volcano_df["Significance"] == "Down-regulated").sum() - st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") - - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") - with col2: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") + st.info("No data available for volcano plot.") + st.stop() + +volcano_df = pivot_df.copy() +volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"]) + +volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) + +fc_thresh = st.slider( + "log2 Fold Change threshold", + min_value=0.5, + max_value=3.0, + value=1.0, + step=0.1, +) + +p_thresh = st.slider( + "p-adj (FDR) threshold", + min_value=0.001, + max_value=0.1, + value=0.05, + step=0.001, +) + +volcano_df["Significance"] = "Not significant" +volcano_df.loc[ + (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), + "Significance", +] = "Up-regulated" + +volcano_df.loc[ + (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), + "Significance", +] = "Down-regulated" + +fig_volcano = px.scatter( + volcano_df, + x="log2FC", + y="neg_log10_padj", + color="Significance", + hover_data=["ProteinName", "log2FC", "p-value", "p-adj"], + color_discrete_map={ + "Up-regulated": "red", + "Down-regulated": "blue", + "Not significant": "lightgrey", + } +) + +fig_volcano.add_vline(x=fc_thresh, line_dash="dash") +fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") +fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") + +# Make x-axis symmetric around zero +max_abs_fc = volcano_df["log2FC"].abs().max() +x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding + +fig_volcano.update_layout( + xaxis_title="log2 Fold Change", + yaxis_title="-log10(p-adj)", + xaxis_range=x_range, + height=600, +) + +st.plotly_chart(fig_volcano, use_container_width=True) + +up_count = (volcano_df["Significance"] == "Up-regulated").sum() +down_count = (volcano_df["Significance"] == "Down-regulated").sum() +st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") +with col2: + st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index ebf7e65..4abbf92 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -47,29 +47,6 @@ def configure(self) -> None: self.ui.select_input_file("mzML-files", multiple=True, reactive=True) self.ui.select_input_file("fasta-file", multiple=False) - self.params = self.parameter_manager.get_parameters_from_json() - saved_mode = self.params.get("analysis-mode", "LFQ") - - self.ui.input_widget( - key="analysis-mode", - default=saved_mode, - name="Analysis Mode", - widget_type="selectbox", - options=["LFQ", "TMT"], - help="Choose between Label-Free Quantification (LFQ) or Tandem Mass Tag (TMT) analysis.", - reactive=True - ) - - self.params = self.parameter_manager.get_parameters_from_json() - current_mode = self.params.get("analysis-mode", "LFQ") - - if current_mode == "LFQ": - self.render_lfq_tabs() - else: - self.render_tmt_tabs() - - def render_lfq_tabs(self): - st.subheader("LFQ Analysis Mode") t = st.tabs(["**Identification**", "**Rescoring**", "**Filtering**", "**Library Generation**", "**Quantification**", "**Group Selection**"]) with t[0]: @@ -93,10 +70,10 @@ def render_lfq_tabs(self): st.info(""" **Decoy Database Settings:** * **method**: How decoy sequences are generated from target protein sequences. - *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly - rearranges the amino acids. Both methods preserve the amino acid composition - of the original protein, ensuring decoys have similar properties to real sequences - for accurate false discovery rate (FDR) estimation. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. """) self.ui.input_TOPP( "DecoyDatabase", @@ -125,7 +102,7 @@ def render_lfq_tabs(self): st.info(comet_info) comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] if not self.params.get("generate-decoys", True): # Only show decoy_string when not generating decoys comet_include.append("PeptideIndexing:decoy_string") @@ -134,7 +111,7 @@ def render_lfq_tabs(self): "CometAdapter", custom_defaults={ "threads": 8, - "instrument": "low_res", + "instrument": "high_res", "missed_cleavages": 2, "min_peptide_length": 6, "max_peptide_length": 40, @@ -143,19 +120,16 @@ def render_lfq_tabs(self): "isotope_error": "0/1", "precursor_charge": "2:4", "precursor_mass_tolerance": 20.0, - "fragment_mass_tolerance": 0.6, - "fragment_bin_offset": 0.4, + "fragment_mass_tolerance": 0.02, + "fragment_bin_offset": 0.0, "max_variable_mods_in_peptide": 3, "minimum_peaks": 1, "clip_nterm_methionine": "true", - "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)", - "PeptideIndexing:IL_equivalent": True, + "PeptideIndexing:IL_equivalent": "true", "PeptideIndexing:unmatched_action": "warn", "PeptideIndexing:decoy_string": "rev_", - "mass_recalibration": False, }, include_parameters=comet_include, - flag_parameters=["PeptideIndexing:IL_equivalent", "mass_recalibration"], exclude_parameters=["second_enzyme"], ) @@ -176,10 +150,9 @@ def render_lfq_tabs(self): "subset_max_train": 300000, "decoy_pattern": "rev_", "score_type": "pep", - "post_processing_tdc": True, + "post_processing_tdc": "true", }, include_parameters=percolator_include, - flag_parameters=["post_processing_tdc"], exclude_parameters=["out_type"], ) @@ -275,10 +248,6 @@ def render_lfq_tabs(self): "psmFDR": 0.01, "proteinFDR": 0.01, "picked_proteinFDR": "true", - "alignment_order": "star", - "protein_quantification": "unique_peptides", - "quantification_method": "feature_intensity", - "protein_inference": "aggregation", }, include_parameters=["intThreshold", "psmFDR", "proteinFDR"], ) @@ -329,295 +298,6 @@ def render_lfq_tabs(self): if orphaned_keys: self.parameter_manager.save_parameters() - def render_tmt_tabs(self): - st.subheader("TMT Analysis Mode") - # Create tabs for different analysis steps. - t = st.tabs( - ["**IsobaricAnalyzer**", "**CometAdapter**", "**PercolatorAdapter**", "**IDFilter**", "**IDMapper**", "**FileMerger**", - "**ProteinInference**", "**IDFilter**", "**IDConflictResolver**", "**ProteinQuantifier**", "**Group Selection**"] - ) - with t[0]: - # Checkbox for decoy generation - # reactive=True ensures the parent configure() fragment re-runs when checkbox changes, - # so conditional UI (DecoyDatabase settings) updates immediately - self.ui.input_widget( - key="generate-decoys", - default=True, - name="Generate Decoy Database", - widget_type="checkbox", - help="Generate reversed decoy sequences for FDR calculation. Disable if your FASTA already contains decoys.", - reactive=True, - ) - - # Reload params to get current checkbox value after it was saved - self.params = self.parameter_manager.get_parameters_from_json() - - # Show DecoyDatabase settings if generating decoys - if self.params.get("generate-decoys", True): - st.info(""" - **Decoy Database Settings:** - * **method**: How decoy sequences are generated from target protein sequences. - *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly - rearranges the amino acids. Both methods preserve the amino acid composition - of the original protein, ensuring decoys have similar properties to real sequences - for accurate false discovery rate (FDR) estimation. - """) - self.ui.input_TOPP( - "DecoyDatabase", - custom_defaults={ - "decoy_string": "rev_", - "decoy_string_position": "prefix", - "method": "reverse", - }, - include_parameters=["method"], - ) - - comet_info = """ - **Identification (Comet):** - * **enzyme**: The enzyme used for peptide digestion. - * **missed_cleavages**: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. - * **fixed_modifications**: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' - * **variable_modifications**: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' - * **instrument**: Type of instrument (high_res or low_res). Use 'high_res' for high-resolution MS2 (Orbitrap, TOF), 'low_res' for ion trap. - * **fragment_mass_tolerance**: Fragment mass tolerance for MS2 matching. - * **fragment_bin_offset**: Offset for binning MS2 spectra. Typically 0.0 for high-res, 0.4 for low-res instruments. - """ - if not self.params.get("generate-decoys", True): - comet_info += """* **PeptideIndexing:decoy_string**: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions - in the protein database to indicate decoy proteins. - """ - st.info(comet_info) - - st.write(Path(self.workflow_dir, "results")) - - comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] - if not self.params.get("generate-decoys", True): - # Only show decoy_string when not generating decoys - comet_include.append("PeptideIndexing:decoy_string") - - self.ui.input_TOPP( - "IsobaricAnalyzer", - custom_defaults={ - "tmt11plex:reference_channel": 126, - "type": "tmt11plex", - "extraction:select_activation": "auto", - "extraction:reporter_mass_shift": 0.002, - "extraction:min_reporter_intensity": 0.0, - "extraction:min_precursor_purity": 0.0, - "extraction:precursor_isotope_deviation": 10.0, - "quantification:isotope_correction": "false", - }, - tool_instance_name="IsobaricAnalyzer-TMT", - ) - with t[1]: - comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset", "PeptideIndexing:IL_equivalent"] - self.ui.input_TOPP( - "CometAdapter", - custom_defaults={ - "PeptideIndexing:IL_equivalent": True, - "clip_nterm_methionine": "true", - "instrument": "high_res", - "missed_cleavages": 2, - "min_peptide_length": 6, - "max_peptide_length": 40, - "enzyme": "Trypsin/P", - "PeptideIndexing:unmatched_action": "warn", - "max_variable_mods_in_peptide": 3, - "precursor_mass_tolerance": 4.5, - "isotope_error": "0/1", - "precursor_error_units": "ppm", - "num_hits": 1, - "num_enzyme_termini": "fully", - "fragment_bin_offset": 0.0, - "minimum_peaks": 10, - "precursor_charge": "2:4", - "fragment_mass_tolerance": 0.015, - "PeptideIndexing:unmatched_action": "warn", - "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)\nTMT6plex (K)\nTMT6plex (N-term)", - "debug": 0, - "force": True, - }, - include_parameters=comet_include, - flag_parameters=["PeptideIndexing:IL_equivalent", "force"], - exclude_parameters=["second_enzyme"], - tool_instance_name="CometAdapter-TMT", - ) - with t[2]: - st.info(""" - **Filtering (IDFilter):** - * **score:type_peptide**: Score used for filtering. If empty, the main score is used. - * **score:psm**: The score which should be reached by a peptide hit to be kept. (use 'NAN' to disable this filter) - """) - self.ui.input_TOPP( - "PercolatorAdapter", - custom_defaults={ - "subset_max_train": 300000, - "decoy_pattern": "DECOY_", - "score_type": "pep", - "post_processing_tdc": True, - "debug": 0, - }, - flag_parameters=["post_processing_tdc"], - tool_instance_name="PercolatorAdapter-TMT", - ) - - with t[3]: - self.ui.input_TOPP( - "IDFilter", - custom_defaults={ - "score:type_peptide": "q-value", - "score:psm": 0.10, - }, - tool_instance_name="IDFilter-strict", - ) - with t[4]: - st.info(""" - **Quantification (ProteomicsLFQ):** - * **intThreshold**: Peak intensity threshold applied in seed detection. - * **psmFDR**: FDR threshold for sub-protein level (e.g. 0.05=5%). Use -FDR_type to choose the level. Cutoff is applied at the highest level. If Bayesian inference was chosen, it is equivalent with a peptide FDR - * **proteinFDR**: Protein FDR threshold (0.05=5%). - """) - self.ui.input_TOPP( - "IDMapper", - custom_defaults={ - "threads": 8, - "debug": 0, - }, - tool_instance_name="IDMapper-TMT", - ) - with t[5]: - self.ui.input_TOPP( - "FileMerger", - custom_defaults={ - "in_type": "consensusXML", - "append_method": "append_cols", - "annotate_file_origin": True, - "threads": 8, - }, - flag_parameters=["annotate_file_origin"], - tool_instance_name="FileMerger-TMT", - ) - with t[6]: - self.ui.input_TOPP( - "ProteinInference", - custom_defaults={ - "threads": 8, - "picked_decoy_string": "DECOY_", - "picked_fdr": "true", - "protein_fdr": "true", - "Algorithm:use_shared_peptides": "true", - "Algorithm:annotate_indistinguishable_groups": "true", - "Algorithm:score_type": "PEP", - "Algorithm:score_aggregation_method": "best", - "Algorithm:min_peptides_per_protein": 1, - }, - tool_instance_name="ProteinInference-TMT", - ) - with t[7]: - # A single checkbox widget for workflow logic. - # self.ui.input_widget("run-python-script", False, "Run custom Python script") * - # Generate input widgets for a custom Python tool, located at src/python-tools. - # Parameters are specified within the file in the DEFAULTS dictionary. - # self.ui.input_python("example") * - self.ui.input_TOPP( - "IDFilter", - custom_defaults={ - "score:type_protein": "q-value", - "score:proteingroup": 0.01, - "score:psm": 0.01, - "delete_unreferenced_peptide_hits": True, - "remove_decoys": True - }, - flag_parameters=["delete_unreferenced_peptide_hits", "remove_decoys"], - tool_instance_name="IDFilter-lenient", - ) - with t[8]: - self.ui.input_TOPP( - "IDConflictResolver", - custom_defaults={ - "threads": 4, - }, - tool_instance_name="IDConflictResolver-TMT", - ) - - with t[9]: - self.ui.input_TOPP( - "ProteinQuantifier", - custom_defaults={ - "method": "top", - "top:N": 3, - "top:aggregate": "median", - "top:include_all": True, - "ratios": True, - "threads": 8, - "debug": 0, - }, - flag_parameters=["top:include_all", "ratios"], - tool_instance_name="ProteinQuantifier-TMT", - ) - with t[10]: - st.markdown("### πŸ§ͺ TMT Sample Group Assignment") - - # 1. Determine TMT type (e.g., tmt10plex, tmt16plex) - target_key = f"{self.parameter_manager.topp_param_prefix}IsobaricAnalyzer:1:type" - selected_tmt = st.session_state.get(target_key, "tmt12plex") - - if "tmt" in selected_tmt: - import re - # Extract the number to determine the plex count - num_plex_match = re.search(r'\d+', selected_tmt) - if num_plex_match: - num_plex = int(num_plex_match.group()) - all_channels = [f"sample{i+1}" for i in range(num_plex)] - - st.info( - "Enter a group name for each TMT channel.\n\n" - "Type **'skip'** for channels you wish to skip. (e.g., control, case, skip)" - ) - - # 2. Create an input_widget for each channel (automatically saved to params.json) - cols = st.columns(2) - for i, ch in enumerate(all_channels): - with cols[i % 2]: - self.ui.input_widget( - key=f"TMT-group-{ch}", - default="", - name=f"Group for {ch}", - widget_type="text", - help="Enter group name or 'skip' to ignore this channel.", - ) - - # 3. Read values from params.json and construct a dictionary in tmt_group_map format - # (This can be used later to filter DataFrames in subsequent logic) - self.params = self.parameter_manager.get_parameters_from_json() - - tmt_group_map = {} - for i, ch in enumerate(all_channels): - # Retrieve stored value (default is empty string) - group_val = self.params.get(f"TMT-group-{ch}", "") - tmt_group_map[str(i)] = group_val - - # For data inspection (remove if not needed) - if st.checkbox("Show current TMT mapping"): - st.json(tmt_group_map) - - # 4. Clean up parameters from unused/previous TMT settings - all_possible_channel_keys = {f"TMT-group-{ch}" for ch in all_channels} - orphaned_keys = [ - k for k in self.params.keys() - if k.startswith("TMT-group-") and k not in all_possible_channel_keys - ] - - if orphaned_keys: - for key in orphaned_keys: - del self.params[key] - self.parameter_manager.save_parameters() - - else: - st.warning("Please select a TMT type in the parameters first.") - def execution(self) -> bool: """ Refactored TOPP workflow execution: @@ -670,944 +350,639 @@ def execution(self) -> bool: st.info(f"Using original FASTA: {fasta_path.name}") database_fasta = fasta_path - current_mode = self.params.get("analysis-mode", "LFQ") - st.write(f"Current analysis mode: **{current_mode}**") - - if current_mode == "LFQ": - self.logger.log("βš™οΈ Running LFQ workflow") - - # ================================ - # 1️⃣ Directory setup - # ================================ - results_dir = Path(self.workflow_dir, "results") - comet_dir = results_dir / "comet_results" - perc_dir = results_dir / "percolator_results" - filter_dir = results_dir / "psm_filter" - quant_dir = results_dir / "quant_results" - - results_dir = Path(self.workflow_dir, "input-files") - - for d in [comet_dir, perc_dir, filter_dir, quant_dir]: - d.mkdir(parents=True, exist_ok=True) - - self.logger.log("πŸ“ Output directories created") - - # ================================ - # 2️⃣ File path definitions (per sample) - # ================================ - comet_results = [] - percolator_results = [] - filter_results = [] - - for mz in in_mzML: - stem = Path(mz).stem - comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) - percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) - filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) - - # ================================ - # 3️⃣ Per-file processing - # ================================ - for i, mz in enumerate(in_mzML): - stem = Path(mz).stem - st.info(f"Processing sample: {stem}") - - self.logger.log("πŸ”¬ Starting per-sample processing...") - - # --- CometAdapter --- - self.logger.log("πŸ”Ž Running peptide search...") - with st.spinner(f"CometAdapter ({stem})"): - comet_extra_params = {"database": str(database_fasta)} - if self.params.get("generate-decoys", True): - # Propagate decoy_string from DecoyDatabase - comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - - if not self.executor.run_topp( - "CometAdapter", - { - "in": in_mzML, - "out": comet_results, - }, - comet_extra_params, - ): - self.logger.log("Workflow stopped due to error") - return False + # ================================ + # 1️⃣ Directory setup + # ================================ + results_dir = Path(self.workflow_dir, "results") + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + filter_dir = results_dir / "filter_results" + quant_dir = results_dir / "quant_results" + + for d in [comet_dir, perc_dir, filter_dir, quant_dir]: + d.mkdir(parents=True, exist_ok=True) + + self.logger.log("πŸ“ Output directories created") + + # # ================================ + # # 2️⃣ File path definitions (per sample) + # # ================================ + comet_results = [] + percolator_results = [] + filter_results = [] + + for mz in in_mzML: + stem = Path(mz).stem + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) + filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) - # Get fragment tolerance from CometAdapter parameters for visualization - comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") - frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) - frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" - - # Build visualization cache for Comet results - results_dir_path = Path(self.workflow_dir, "results") - cache_dir = results_dir_path / "insight_cache" - cache_dir.mkdir(parents=True, exist_ok=True) - - # Get mzML directory - mzml_dir = Path(in_mzML[0]).parent - - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} - - for idxml_file in comet_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # ================================ + # 3️⃣ Per-file processing + # ================================ + for i, mz in enumerate(in_mzML): + stem = Path(mz).stem + st.info(f"Processing sample: {stem}") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + self.logger.log("πŸ”¬ Starting per-sample processing...") - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + ): + self.logger.log("Workflow stopped due to error") + return False + + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - self.logger.log("βœ… Peptide search complete") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # if not Path(comet_results).exists(): - # st.error(f"CometAdapter failed for {stem}") - # st.stop() + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # --- PercolatorAdapter --- - self.logger.log("πŸ“Š Running rescoring...") - with st.spinner(f"PercolatorAdapter ({stem})"): - if not self.executor.run_topp( - "PercolatorAdapter", - { - "in": comet_results, - "out": percolator_results, - }, - {"decoy_pattern": decoy_string}, # Always propagated from upstream - ): - self.logger.log("Workflow stopped due to error") - return False + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - # Build visualization cache for Percolator results - for idxml_file in percolator_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + self.logger.log("βœ… Peptide search complete") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter ({stem})"): + if not self.executor.run_topp( + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + {"decoy_pattern": decoy_string}, # Always propagated from upstream + ): + self.logger.log("Workflow stopped due to error") + return False + + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - self.logger.log("βœ… Rescoring complete") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - # if not Path(percolator_results[i]).exists(): - # st.error(f"PercolatorAdapter failed for {stem}") - # st.stop() + self.logger.log("βœ… Rescoring complete") - # --- IDFilter --- - self.logger.log("πŸ”§ Filtering identifications...") - with st.spinner(f"IDFilter ({stem})"): - if not self.executor.run_topp( - "IDFilter", - { - "in": percolator_results, - "out": filter_results, - }, - ): - self.logger.log("Workflow stopped due to error") - return False + # if not Path(percolator_results[i]).exists(): + # st.error(f"PercolatorAdapter failed for {stem}") + # st.stop() - # Build visualization cache for Filter results - for idxml_file in filter_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter ({stem})"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": filter_results, + }, + ): + self.logger.log("Workflow stopped due to error") + return False + + # Build visualization cache for Filter results + for idxml_file in filter_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - self.logger.log("βœ… Filtering complete") + self.logger.log("βœ… Filtering complete") - # if not Path(filter_results[i]).exists(): - # st.error(f"IDFilter failed for {stem}") - # st.stop() + # if not Path(filter_results[i]).exists(): + # st.error(f"IDFilter failed for {stem}") + # st.stop() - # ================================ - # EasyPQP Spectral Library Generation (optional) - # ================================ - if self.params.get("generate-library", False): - self.logger.log("πŸ“š Building spectral library with EasyPQP...") - st.info("Building spectral library with EasyPQP...") - library_dir = Path(self.workflow_dir, "results", "library") - library_dir.mkdir(parents=True, exist_ok=True) - - psms_files, peaks_files = [], [] - - for filter_idxml in filter_results: - original_stem = Path(filter_idxml).stem.replace("_filter", "") - matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) - if not matching_mzml: - self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") - continue - - # easypqp library requires specific extensions for file recognition: - # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension - # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension - # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name - psms_out = str(library_dir / f"{original_stem}.psmpkl") - peaks_out = str(library_dir / f"{original_stem}.peakpkl") - - convert_cmd = [ - "easypqp", "convert", - "--pepxml", filter_idxml, - "--spectra", matching_mzml, - "--psms", psms_out, - "--peaks", peaks_out - ] - if self.executor.run_command(convert_cmd): - psms_files.append(psms_out) - peaks_files.append(peaks_out) - - if psms_files: - # easypqp library outputs TSV format (despite common .pqp extension) - library_tsv = str(library_dir / "spectral_library.tsv") - library_cmd = ["easypqp", "library", "--out", library_tsv] - - if not self.params.get("library-use-fdr", False): - # --nofdr only skips FDR recalculation, NOT threshold filtering - # Set all thresholds to 1.0 to bypass filtering for pre-filtered input - library_cmd.extend([ - "--nofdr", - "--psm_fdr_threshold", "1.0", - "--peptide_fdr_threshold", "1.0", - "--protein_fdr_threshold", "1.0" - ]) - else: - # Apply user-specified FDR filtering - library_cmd.extend([ - "--psm_fdr_threshold", - str(self.params.get("library-psm-fdr", 0.01)), - "--peptide_fdr_threshold", - str(self.params.get("library-peptide-fdr", 0.01)), - "--protein_fdr_threshold", - str(self.params.get("library-protein-fdr", 0.01)) - ]) - - for psms, peaks in zip(psms_files, peaks_files): - library_cmd.extend([psms, peaks]) - - if self.executor.run_command(library_cmd): - self.logger.log("βœ… Spectral library created") - st.success("Spectral library created") - else: - self.logger.log("Warning: Failed to build spectral library") + # ================================ + # EasyPQP Spectral Library Generation (optional) + # ================================ + if self.params.get("generate-library", False): + self.logger.log("πŸ“š Building spectral library with EasyPQP...") + st.info("Building spectral library with EasyPQP...") + library_dir = Path(self.workflow_dir, "results", "library") + library_dir.mkdir(parents=True, exist_ok=True) + + psms_files, peaks_files = [], [] + + for filter_idxml in filter_results: + original_stem = Path(filter_idxml).stem.replace("_filter", "") + matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) + if not matching_mzml: + self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") + continue + + # easypqp library requires specific extensions for file recognition: + # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension + # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension + # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name + psms_out = str(library_dir / f"{original_stem}.psmpkl") + peaks_out = str(library_dir / f"{original_stem}.peakpkl") + + convert_cmd = [ + "easypqp", "convert", + "--pepxml", filter_idxml, + "--spectra", matching_mzml, + "--psms", psms_out, + "--peaks", peaks_out + ] + if self.executor.run_command(convert_cmd): + psms_files.append(psms_out) + peaks_files.append(peaks_out) + + if psms_files: + # easypqp library outputs TSV format (despite common .pqp extension) + library_tsv = str(library_dir / "spectral_library.tsv") + library_cmd = ["easypqp", "library", "--out", library_tsv] + + if not self.params.get("library-use-fdr", False): + # --nofdr only skips FDR recalculation, NOT threshold filtering + # Set all thresholds to 1.0 to bypass filtering for pre-filtered input + library_cmd.extend([ + "--nofdr", + "--psm_fdr_threshold", "1.0", + "--peptide_fdr_threshold", "1.0", + "--protein_fdr_threshold", "1.0" + ]) else: - self.logger.log("Warning: No PSMs converted for library generation") - - st.success(f"βœ“ {stem} identification completed") - - # # ================================ - # # 4️⃣ ProteomicsLFQ (cross-sample) - # # ================================ - self.logger.log("πŸ“ˆ Running cross-sample quantification...") - st.info("Running ProteomicsLFQ (cross-sample quantification)") - - quant_mztab = str(quant_dir / "openms_quant.mzTab") - quant_cxml = str(quant_dir / "openms.consensusXML") - quant_msstats = str(quant_dir / "openms_msstats.csv") - - with st.spinner("ProteomicsLFQ"): - combined_in = " ".join(in_mzML) - combined_ids = " ".join(filter_results) - self.logger.log(f"COMBINED_IN {combined_in}", 1) - self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) - self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) - self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) - - # βœ… Streamlit output (debug view) - st.markdown("### πŸ” ProteomicsLFQ Input Debug") - st.write("**combined_in:**", combined_in) - st.write("**combined_in type:**", type(combined_in).__name__) - - st.write("**combined_ids:**", combined_ids) - st.write("**combined_ids type:**", type(combined_ids).__name__) - - if not self.executor.run_topp( - "ProteomicsLFQ", - { - "in": [in_mzML], - "ids": [filter_results], - "out": [quant_mztab], - "out_cxml": [quant_cxml], - "out_msstats": [quant_msstats], - }, - { - "fasta": str(database_fasta), - "threads": 12, - # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 - "PeptideQuantification:extract:IM_window": "0.0", - "PeptideQuantification:faims:merge_features": "false", - }, - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… Quantification complete") - - # if not Path(quant_mztab).exists(): - # st.error("ProteomicsLFQ failed: mzTab not created") - # st.stop() - - # ================================ - # 5️⃣ Final report - # # ================================ - st.success("πŸŽ‰ TOPP workflow completed successfully") - st.write("πŸ“ Results directory:") - st.code(str(results_dir)) - - st.write("πŸ“„ Generated files:") - st.write(f"- mzTab: {quant_mztab}") - st.write(f"- consensusXML: {quant_cxml}") - st.write(f"- MSstats CSV: {quant_msstats}") - - return True - else: - self.logger.log("βš™οΈ Running TMT workflow") - - results_dir = Path(self.workflow_dir, "results") - iso_dir = results_dir / "isobaric_consensusXML" - comet_dir = results_dir / "comet_results" - perc_dir = results_dir / "percolator_results" - psm_filter_dir = results_dir / "psm_filter" - map_dir = results_dir / "idmapper" - merge_dir = results_dir / "merged" - protein_dir = results_dir / "protein" - msstats_dir = results_dir / "msstats" - quant_dir = results_dir / "quant_results" - - iso_consensus = [] - comet_results = [] - percolator_results = [] - psm_filtered = [] - mapped_ids = [] - - for d in [ - iso_dir, comet_dir, perc_dir, psm_filter_dir, - map_dir, merge_dir, protein_dir, msstats_dir, quant_dir - ]: - d.mkdir(parents=True, exist_ok=True) - - for mz in in_mzML: - stem = Path(mz).stem - iso_consensus.append(str(iso_dir / f"{stem}_iso.consensusXML")) - comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) - percolator_results.append(str(perc_dir / f"{stem}_comet_perc.idXML")) - psm_filtered.append(str(psm_filter_dir / f"{stem}_comet_perc_filter.idXML")) - mapped_ids.append(str(map_dir / f"{stem}_comet_perc_filter_map.consensusXML")) - - merged_id = str(merge_dir / "ID_mapper_merge.consensusXML") - protein_id = str(protein_dir / "ID_mapper_merge_epi.consensusXML") - protein_filter = str(protein_dir / "ID_mapper_merge_epi_filter.consensusXML") - protein_resolved = str(protein_dir / "ID_mapper_merge_epi_filter_resconf.consensusXML") - consensus_out = str(quant_dir / "openms_design_protein_openms.csv") - - # --- IsobaricAnalyzer --- - self.logger.log("🏷️ Running isobaric labeling analysis...") - with st.spinner("IsobaricAnalyzer"): - if not self.executor.run_topp( - "IsobaricAnalyzer", - { - "in": in_mzML, - "out": iso_consensus, - }, - tool_instance_name="IsobaricAnalyzer-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… IsobaricAnalyzer complete") - - # --- CometAdapter --- - self.logger.log("πŸ”Ž Running peptide search...") - with st.spinner(f"CometAdapter ({stem})"): - comet_extra_params = {"database": str(database_fasta)} - if self.params.get("generate-decoys", True): - # Propagate decoy_string from DecoyDatabase - comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - if not self.executor.run_topp( - "CometAdapter", - { - "in": in_mzML, - "out": comet_results, - }, - comet_extra_params, - tool_instance_name="CometAdapter-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… CometAdapter complete") - - # Get fragment tolerance from CometAdapter parameters for visualization - comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") - frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) - frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" - - # Build visualization cache for Comet results - results_dir_path = Path(self.workflow_dir, "results") - cache_dir = results_dir_path / "insight_cache" - cache_dir.mkdir(parents=True, exist_ok=True) - - # Get mzML directory - mzml_dir = Path(in_mzML[0]).parent - - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} - - for idxml_file in comet_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # Apply user-specified FDR filtering + library_cmd.extend([ + "--psm_fdr_threshold", + str(self.params.get("library-psm-fdr", 0.01)), + "--peptide_fdr_threshold", + str(self.params.get("library-peptide-fdr", 0.01)), + "--protein_fdr_threshold", + str(self.params.get("library-protein-fdr", 0.01)) + ]) + + for psms, peaks in zip(psms_files, peaks_files): + library_cmd.extend([psms, peaks]) + + if self.executor.run_command(library_cmd): + self.logger.log("βœ… Spectral library created") + st.success("Spectral library created") + else: + self.logger.log("Warning: Failed to build spectral library") + else: + self.logger.log("Warning: No PSMs converted for library generation") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + st.success(f"βœ“ {stem} identification completed") - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # ================================ + # 4️⃣ ProteomicsLFQ (cross-sample) + # ================================ + self.logger.log("πŸ“ˆ Running cross-sample quantification...") + st.info("Running ProteomicsLFQ (cross-sample quantification)") - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + quant_mztab = str(quant_dir / "openms_quant.mzTab") + quant_cxml = str(quant_dir / "openms.consensusXML") + quant_msstats = str(quant_dir / "openms_msstats.csv") - self.logger.log("βœ… Peptide search complete") + with st.spinner("ProteomicsLFQ"): + combined_in = " ".join(in_mzML) + combined_ids = " ".join(filter_results) + self.logger.log(f"COMBINED_IN {combined_in}", 1) + self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) + self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) + self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) - # --- PercolatorAdapter --- - self.logger.log("πŸ“Š Running rescoring...") - with st.spinner(f"PercolatorAdapter"): - if not self.executor.run_topp( - "PercolatorAdapter", - { - "in": comet_results, - "out": percolator_results, - }, - tool_instance_name="PercolatorAdapter-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - # Build visualization cache for Percolator results - for idxml_file in percolator_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # βœ… Streamlit output (debug view) + st.markdown("### πŸ” ProteomicsLFQ Input Debug") + st.write("**combined_in:**", combined_in) + st.write("**combined_in type:**", type(combined_in).__name__) - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) - - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + st.write("**combined_ids:**", combined_ids) + st.write("**combined_ids type:**", type(combined_ids).__name__) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) - - self.logger.log("βœ… PercolatorAdapter complete") - - # --- IDFilter --- - self.logger.log("πŸ”§ Filtering identifications...") - with st.spinner(f"IDFilter"): if not self.executor.run_topp( - "IDFilter", - { - "in": percolator_results, - "out": psm_filtered, - }, - tool_instance_name="IDFilter-strict" - ): + "ProteomicsLFQ", + { + "in": [in_mzML], + "ids": [filter_results], + "out": [quant_mztab], + "out_cxml": [quant_cxml], + "out_msstats": [quant_msstats], + }, + { + "fasta": str(database_fasta), + "psmFDR": 0.5, + "proteinFDR": 0.5, + "threads": 12, + # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 + "PeptideQuantification:extract:IM_window": "0.0", + "PeptideQuantification:faims:merge_features": "false", + } + ): self.logger.log("Workflow stopped due to error") return False - self.logger.log("βœ… IDFilter-strict complete") - - # Build visualization cache for Filter results - for idxml_file in psm_filtered: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + self.logger.log("βœ… Quantification complete") + + # ====================================================== + # ⚠️ 5️⃣ GO Enrichment Analysis (INLINE IN EXECUTION) + # ====================================================== + workspace_path = Path(self.workflow_dir).parent + res = get_abundance_data(workspace_path) + if res is not None: + pivot_df, _, _ = res + self.logger.log("βœ… pivot_df loaded, starting GO enrichment...") + self._run_go_enrichment(pivot_df, results_dir) + else: + st.warning("GO enrichment skipped: abundance data not available.") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # ================================ + # 5️⃣ Final report + # # ================================ + st.success("πŸŽ‰ TOPP workflow completed successfully") + st.write("πŸ“ Results directory:") + st.code(str(results_dir)) + + return True + + def _run_go_enrichment(self, pivot_df: pd.DataFrame, results_dir: Path): + p_cutoff = 0.05 + fc_cutoff = 1.0 + + analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() + + if analysis_df.empty: + st.error("No valid statistical data found for GO enrichment.") + self.logger.log("❗ analysis_df is empty") + else: + with st.spinner("Fetching GO terms from MyGene.info API..."): + mg = mygene.MyGeneInfo() + + def get_clean_uniprot(name): + parts = str(name).split("|") + return parts[1] if len(parts) >= 2 else parts[0] + + analysis_df["UniProt"] = analysis_df["ProteinName"].apply(get_clean_uniprot) + + bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() + fg_ids = analysis_df[ + (analysis_df["p-value"] < p_cutoff) & + (analysis_df["log2FC"].abs() >= fc_cutoff) + ]["UniProt"].dropna().astype(str).unique().tolist() + self.logger.log("βœ… get_clean_uniprot applied") + + if len(fg_ids) < 3: + st.warning( + f"Not enough significant proteins " + f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " + f"Found: {len(fg_ids)}" + ) + self.logger.log("❗ Not enough significant proteins") + else: + res_list = mg.querymany( + bg_ids, scopes="uniprot", fields="go", as_dataframe=False + ) + res_go = pd.DataFrame(res_list) + if "notfound" in res_go.columns: + res_go = res_go[res_go["notfound"] != True] + + def extract_go_terms(go_data, go_type): + if not isinstance(go_data, dict) or go_type not in go_data: + return [] + terms = go_data[go_type] + if isinstance(terms, dict): + terms = [terms] + return list({t.get("term") for t in terms if "term" in t}) + + for go_type in ["BP", "CC", "MF"]: + res_go[f"{go_type}_terms"] = res_go["go"].apply( + lambda x: extract_go_terms(x, go_type) + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + annotated_ids = set(res_go["query"].astype(str)) + fg_set = annotated_ids.intersection(fg_ids) + bg_set = annotated_ids + self.logger.log(f"βœ… fg_set bg_set are set") + + def run_go(go_type): + go2fg = defaultdict(set) + go2bg = defaultdict(set) + + for _, row in res_go.iterrows(): + uid = str(row["query"]) + for term in row[f"{go_type}_terms"]: + go2bg[term].add(uid) + if uid in fg_set: + go2fg[term].add(uid) + + records = [] + N_fg = len(fg_set) + N_bg = len(bg_set) + + for term, fg_genes in go2fg.items(): + a = len(fg_genes) + if a == 0: + continue + b = N_fg - a + c = len(go2bg[term]) - a + d = N_bg - (a + b + c) + + _, p = fisher_exact([[a, b], [c, d]], alternative="greater") + records.append({ + "GO_Term": term, + "Count": a, + "GeneRatio": f"{a}/{N_fg}", + "p_value": p, + }) + + df = pd.DataFrame(records) + if df.empty: + return None, None + + df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) + df = df.sort_values("p_value").head(20) + + # βœ… Plotly Figure + fig = px.bar( + df, + x="-log10(p)", + y="GO_Term", + orientation="h", + title=f"GO Enrichment ({go_type})", + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + self.logger.log(f"βœ… Plotly Figure generated") - # --- IDMapper --- - self.logger.log("πŸ—ΊοΈ Mapping IDs to isobaric consensus features...") - for iso, psm, mapped in zip(iso_consensus, psm_filtered, mapped_ids): - iso_stem = Path(iso).stem - with st.spinner(f"IDMapper ({iso_stem})"): - if not self.executor.run_topp( - "IDMapper", - { - "in": [iso], - "id": [psm], - "out": [mapped], - }, - tool_instance_name="IDMapper-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… IDMapper complete") + fig.update_layout( + yaxis=dict(autorange="reversed"), + height=500, + margin=dict(l=10, r=10, t=40, b=10), + ) - # --- FileMerger --- - self.logger.log("πŸ”— Merging mapped consensus files...") - with st.spinner("FileMerger"): - if not self.executor.run_topp( - "FileMerger", - { - "in": mapped_ids, - "out": [merged_id], - }, - tool_instance_name="FileMerger-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… FileMerger complete") + return fig, df - # --- ProteinInference --- - self.logger.log("🧩 Running protein inference...") - with st.spinner("ProteinInference"): - if not self.executor.run_topp( - "ProteinInference", - { - "in": [merged_id], - "out": [protein_id], - }, - tool_instance_name="ProteinInference-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… ProteinInference complete") + go_results = {} - # --- IDFilter-lenient (Protein) --- - self.logger.log("πŸ”¬ Filtering proteins...") - with st.spinner("IDFilter (Protein)"): - if not self.executor.run_topp( - "IDFilter", - { - "in": [protein_id], - "out": [protein_filter], - }, - tool_instance_name="IDFilter-lenient" - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… IDFilter-lenient (Protein) complete") + for go_type in ["BP", "CC", "MF"]: + fig, df_go = run_go(go_type) + if fig is not None: + go_results[go_type] = { + "fig": fig, + "df": df_go + } + self.logger.log(f"βœ… go_type generated") - # ================================ - # ✨ NEW: 8️⃣ IDConflictResolver (protein_filter β†’ protein_resolved) - # ================================ - self.logger.log("βš–οΈ Resolving ID conflicts...") - with st.spinner("IDConflictResolver"): - if not self.executor.run_topp( - "IDConflictResolver", - { - "in": [protein_filter], - "out": [protein_resolved], - }, - tool_instance_name="IDConflictResolver-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… IDConflictResolver complete") + go_dir = results_dir / "go-terms" + go_dir.mkdir(parents=True, exist_ok=True) - # ================================ - # ✨ NEW: πŸ”Ÿ ProteinQuantifier (protein_resolved β†’ consensus_out) - # ================================ - self.logger.log("πŸ“ Running protein quantification...") - with st.spinner("ProteinQuantifier"): - if not self.executor.run_topp( - "ProteinQuantifier", - { - "in": [protein_resolved], - "out": [consensus_out], - }, - tool_instance_name="ProteinQuantifier-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… ProteinQuantifier complete") - self.logger.log("πŸ“„ Generating protein table...") - - self.logger.log("πŸŽ‰ WORKFLOW FINISHED") + import json + go_data = {} + + for go_type in ["BP", "CC", "MF"]: + if go_type in go_results: + fig = go_results[go_type]["fig"] + df = go_results[go_type]["df"] + + go_data[go_type] = { + "fig_json": fig.to_json(), # Figure β†’ JSON string + "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts + } + + go_json_file = go_dir / "go_results.json" + with open(go_json_file, "w") as f: + json.dump(go_data, f) + st.session_state["go_results"] = go_results + st.session_state["go_ready"] = True if go_data else False + self.logger.log("βœ… GO enrichment analysis complete") + @st.fragment def results(self) -> None: diff --git a/src/common/results_helpers.py b/src/common/results_helpers.py index 02d7bd4..db3e103 100644 --- a/src/common/results_helpers.py +++ b/src/common/results_helpers.py @@ -9,7 +9,6 @@ from pyopenms import IdXMLFile, MSExperiment, MzMLFile from src.workflow.ParameterManager import ParameterManager from statsmodels.stats.multitest import multipletests -from statsmodels.stats.multitest import multipletests def get_workflow_dir(workspace): """Get the workflow directory path.""" @@ -198,289 +197,111 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: workflow_dir = get_workflow_dir(Path(workspace_path)) quant_dir = workflow_dir / "results" / "quant_results" - parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - - workflow_params = parameter_manager.get_parameters_from_json() - analysis_mode = workflow_params.get("analysis-mode", "LFQ") + if not quant_dir.exists(): + return None - if analysis_mode == "LFQ": - if not quant_dir.exists(): - return None + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None - csv_files = sorted(quant_dir.glob("*.csv")) - if not csv_files: - return None + csv_file = csv_files[0] - csv_file = csv_files[0] + try: + df = pd.read_csv(csv_file) + except Exception: + return None - try: - df = pd.read_csv(csv_file) - except Exception: - return None + if df.empty: + return None - if df.empty: - return None + # Get group mapping from parameters + param_manager = ParameterManager(workflow_dir) + params = param_manager.get_parameters_from_json() + group_map = { + key[11:]: value # Remove "mzML-group-" prefix + for key, value in params.items() + if key.startswith("mzML-group-") and value + } - # Get group mapping from parameters - param_manager = ParameterManager(workflow_dir) - params = param_manager.get_parameters_from_json() - group_map = { - key[11:]: value # Remove "mzML-group-" prefix - for key, value in params.items() - if key.startswith("mzML-group-") and value - } + if not group_map: + return None - if not group_map: - return None + df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) + df["Group"] = df["Reference"].map(group_map) + df = df.dropna(subset=["Group"]) - df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) - df["Group"] = df["Reference"].map(group_map) - df = df.dropna(subset=["Group"]) + groups = sorted(df["Group"].unique()) - groups = sorted(df["Group"].unique()) + if len(groups) < 2: + return None - if len(groups) < 2: - return None + group1, group2 = groups[:2] - group1, group2 = groups[:2] + # Compute statistics per protein + stats_rows = [] + for protein, protein_df in df.groupby("ProteinName"): + g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values + g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values - # Compute statistics per protein - stats_rows = [] - for protein, protein_df in df.groupby("ProteinName"): - g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values - g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values + if len(g1_vals) < 2 or len(g2_vals) < 2: + pval = np.nan + else: + _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False) - if len(g1_vals) < 2 or len(g2_vals) < 2: - pval = np.nan - else: - _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False) + mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan + mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan - mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan - mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan + log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan - log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan + stats_rows.append({ + "ProteinName": protein, + "log2FC": log2fc, + "p-value": pval, + }) - stats_rows.append({ - "ProteinName": protein, - "log2FC": log2fc, - "p-value": pval, - }) + stats_df = pd.DataFrame(stats_rows) - stats_df = pd.DataFrame(stats_rows) - - if not stats_df.empty: - mask = stats_df["p-value"].notna() - if mask.any(): - _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh") - stats_df.loc[mask, "p-adj"] = p_adj - else: - stats_df["p-adj"] = np.nan - - # Order samples by group (group2 first, then group1) - sample_group_df = df[["Sample", "Group"]].drop_duplicates() - group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist() - group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist() - all_samples = group2_samples + group1_samples - - # Build pivot table - pivot_list = [] - for protein, group_df in df.groupby("ProteinName"): - peptides = ";".join(group_df["PeptideSequence"].unique()) - intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() - intensity_dict_complete = { - sample: intensity_dict.get(sample, 0) - for sample in all_samples - } - row = { - "ProteinName": protein, - **intensity_dict_complete, - "PeptideSequence": peptides, - } - pivot_list.append(row) - - pivot_df = pd.DataFrame(pivot_list) - pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left") - pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]] - - # Build expression matrix (log2-transformed) - expr_df = pivot_df.set_index("ProteinName")[all_samples] - expr_df = expr_df.replace(0, np.nan) - expr_df = np.log2(expr_df + 1) - expr_df = expr_df.dropna() - - return pivot_df, expr_df, group_map - - else: - if not quant_dir.exists(): - return None - - csv_files = sorted(quant_dir.glob("*.csv")) - if not csv_files: - return None - - csv_file = csv_files[0] - - try: - df = pd.read_csv(csv_file, sep="\t", comment="#", engine="python") - except Exception: - return None - - if df.empty: - return None - - # ratio column removal - df = df.loc[:, ~df.columns.str.contains('ratio', case=False)] - - # exclude_indices = st.session_state.get("tmt_exclude_indices", []) - # group_map = st.session_state.get("tmt_group_map", {}) - # Get group mapping from parameters - parameter_manager = ParameterManager(Path(workflow_dir), "TOPP Workflow") - params = parameter_manager.get_parameters_from_json() - group_map = {} - for key, value in params.items(): - if key.startswith("TMT-group-") and value: - # Extract the numeric part from keys like "TMT-group-sample1" - match = re.search(r'sample(\d+)', key) - if match: - # Subtract 1 to convert to a 0-based index (0, 1, 2...). - # If your samples are already 0-based, remove the -1 adjustment. - index = str(int(match.group(1)) - 1) - group_map[index] = value - - # 1. Extract keys labeled as "skip" from group_map as integer list - exclude_indices = [ - int(k) for k, v in group_map.items() if v.lower() == "skip" - ] - - # 2. Remove "skip" entries from group_map (keep only actual group info) - group_map = { - int(k): v for k, v in group_map.items() if v.lower() != "skip" + if not stats_df.empty: + mask = stats_df["p-value"].notna() + if mask.any(): + _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh") + stats_df.loc[mask, "p-adj"] = p_adj + else: + stats_df["p-adj"] = np.nan + + # Order samples by group (group2 first, then group1) + sample_group_df = df[["Sample", "Group"]].drop_duplicates() + group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist() + group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist() + all_samples = group2_samples + group1_samples + + # Build pivot table + pivot_list = [] + for protein, group_df in df.groupby("ProteinName"): + peptides = ";".join(group_df["PeptideSequence"].unique()) + intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() + intensity_dict_complete = { + sample: intensity_dict.get(sample, 0) + for sample in all_samples + } + row = { + "ProteinName": protein, + **intensity_dict_complete, + "PeptideSequence": peptides, } + pivot_list.append(row) - start_column_offset = 4 - - # st.write("exclude_indices:", exclude_indices) - # st.write("group_map:", group_map) - - if not group_map: - st.warning("⚠️ Group mapping information is missing. Please configure sample groups in the Setup page.") - return None - - if exclude_indices: - # st.write("Current columns:", df.columns.tolist()) - # st.write("Number of columns:", len(df.columns)) - # st.write("Exclude indices:", exclude_indices) - # st.write("Offset:", start_column_offset) - cols_to_drop = [df.columns[i + start_column_offset] for i in exclude_indices] - df_cleaned = df.drop(columns=cols_to_drop) - else: - df_cleaned = df.copy() - - if group_map: - # Create new row data (defaulting to empty strings) - # Create a list with the same length as the column order of df_cleaned - new_row = [""] * len(df_cleaned.columns) - new_row[0] = "Group" - - # Get the column names of the current dataframe as a list - current_cols = df_cleaned.columns.tolist() - original_cols = df.columns.tolist() - - for col_name in current_cols[start_column_offset:]: - # Check the original index position of this column - original_idx = original_cols.index(col_name) - start_column_offset - col_pos = current_cols.index(col_name) - new_row[col_pos] = group_map.get(original_idx, "NA") - - # Insert the row at the top of the dataframe - # Create a new DF and concatenate to prepend the row to existing data - group_df = pd.DataFrame([new_row], columns=df_cleaned.columns) - df_with_groups = pd.concat([group_df, df_cleaned], ignore_index=True) - - # drop_msg = f"{len(exclude_indices)} channels dropped" if exclude_indices else "No channels dropped" - # st.success(f"βœ… {drop_msg} and Group names have been inserted at the top of the data.") - - # st.write("### Data Preview with Group Information") - # st.dataframe(df_with_groups.head(10)) - - if group_map and len(set(group_map.values())) >= 2: - # Prepare data for calculation - # Extract group information from row 0 of df_with_groups (the newly added Group row) - # Actual sample data starts from the 5th column (index 4) - group_info_row = df_with_groups.iloc[0] - - # Get unique group names (excluding NA) - unique_groups = sorted([g for g in set(group_map.values()) if g != "NA"]) - g1_name, g2_name = unique_groups[0], unique_groups[1] - - # Extract numerical data for statistical calculation (from row 1 and column index 4 onwards) - # Convert to numeric type (to prevent calculation errors) - numeric_data = df_with_groups.iloc[1:, 4:].apply(pd.to_numeric, errors='coerce') - - # Column indexing by group - # Categorize columns based on the values in the Group row - g1_cols = [col for col in numeric_data.columns if group_info_row[col] == g1_name] - g2_cols = [col for col in numeric_data.columns if group_info_row[col] == g2_name] - - # Calculate log2FC and p-value for each row - def run_stats(row): - v1 = row[g1_cols].dropna() - v2 = row[g2_cols].dropna() - - # log2FC (Group2 / Group1) - m1, m2 = v1.mean(), v2.mean() - l2fc = np.log2(m2 / m1) if m1 > 0 and m2 > 0 else np.nan - - # p-value (T-test) - if len(v1) > 1 and len(v2) > 1: - _, pval = ttest_ind(v1, v2, equal_var=False) - else: - pval = np.nan - return pd.Series([l2fc, pval]) - - stats_results = numeric_data.apply(run_stats, axis=1) - stats_results.columns = ['log2FC', 'p-value'] - # Add Adjusted p-value (FDR) calculation - if not stats_results['p-value'].isna().all(): - # Select only rows that contain p-values - mask = stats_results['p-value'].notna() - # Apply Benjamini-Hochberg (BH) correction - _, p_adj, _, _ = multipletests(stats_results.loc[mask, 'p-value'], method='fdr_bh') - stats_results.loc[mask, 'p-adj'] = p_adj - else: - stats_results['p-adj'] = np.nan - - # Construct the final dataframe (Based on df_cleaned - excluding the group row) - # Insert calculation results into the 2nd and 3rd column positions - pivot_df = df_cleaned.copy() - pivot_df.insert(1, "log2FC", stats_results['log2FC'].values) - pivot_df.insert(2, "p-value", stats_results['p-value'].values) - pivot_df.insert(3, "p-adj", stats_results['p-adj'].values) - - # st.success(f"Analysis Complete: {g1_name} (n={len(g1_cols)}) vs {g2_name} (n={len(g2_cols)})") - - # Set the first column ('protein') of final_df as the index - protein_col = pivot_df.columns[0] - sample_cols = current_cols[start_column_offset:] # Identify actual sample column names - - # Select sample columns and create a matrix - expr_df = pivot_df.set_index(protein_col)[sample_cols] - - # Replace 0 with NaN (to prevent log transformation errors) - expr_df = expr_df.replace(0, np.nan) - - # Log2 transformation (data normalization) - expr_df = np.log2(expr_df + 1) - - # Remove proteins (rows) with any missing values - expr_df = expr_df.dropna() - - return pivot_df, expr_df, group_map - else: - st.warning("⚠️ At least two distinct groups are required for statistical analysis.") - else: - st.warning("⚠️ No group mapping information is set. Please check the Configure page.") - return None + pivot_df = pd.DataFrame(pivot_list) + pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left") + pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]] + + # Build expression matrix (log2-transformed) + expr_df = pivot_df.set_index("ProteinName")[all_samples] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() + + return pivot_df, expr_df, group_map def get_abundance_data(workspace: Path) -> tuple | None: diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 6a587cd..86f265b 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -216,7 +216,7 @@ def read_stderr(): stdout_thread.join() stderr_thread.join() - def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool_instance_name: str = None) -> bool: + def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> bool: """ Constructs and executes commands for the specified tool OpenMS TOPP tool based on the given input and output configurations. Ensures that all input/output file lists @@ -234,9 +234,6 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool tool (str): The executable name or path of the tool. input_output (dict): A dictionary specifying the input/output parameter names (as key) and their corresponding file paths (as value). custom_params (dict): A dictionary of custom parameters to pass to the tool. - tool_instance_name (str, optional): A unique instance name for this tool - invocation, used for parameter lookup when multiple instances of the - same tool exist. If not provided, defaults to the tool name. Returns: bool: True if all commands succeeded, False if any failed. @@ -245,8 +242,6 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool ValueError: If the lengths of input/output file lists are inconsistent, except for single string inputs. """ - # Use tool_instance_name for parameter lookup, fall back to tool name - params_key = tool_instance_name if tool_instance_name else tool # check input: any input lists must be same length, other items can be a single string # e.g. input_mzML : [list of n mzML files], output_featureXML : [list of n featureXML files], input_database : database.tsv io_lengths = [len(v) for v in input_output.values() if len(v) > 1] @@ -266,13 +261,8 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool commands = [] - # Load merged parameters (_defaults + user overrides) for this tool instance - merged_params = self.parameter_manager.get_merged_params(params_key) - flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {}) - if not flag_map: - flag_map = st.session_state.get("_topp_flag_params", {}) - flag_params = set(flag_map.get(params_key, [])) - + # Load parameters for non-defaults + params = self.parameter_manager.get_parameters_from_json() # Construct commands for each process for i in range(n_processes): command = [tool] @@ -291,56 +281,35 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool # standard case, files was a list of strings, take the file name at index else: command += [value[i]] - # Add merged TOPP tool parameters (_defaults + user overrides) - for k, v in merged_params.items(): - if k in flag_params: - # CLI flag: include "-k" only when enabled - if isinstance(v, str): - is_enabled = v.lower() in {"true", "1", "yes", "on"} - else: - is_enabled = bool(v) - if is_enabled: - command += [f"-{k}"] - continue - # For non-flag parameters, skip entirely if empty. - # Note: 0 and 0.0 are valid values, so use explicit checks. - if v == "" or v is None: - continue - command += [f"-{k}"] - if isinstance(v, str) and "\n" in v: - command += v.split("\n") - elif isinstance(v, bool): - command += [str(v).lower()] - else: - command += [str(v)] + # Add non-default TOPP tool parameters + if tool in params.keys(): + for k, v in params[tool].items(): + command += [f"-{k}"] + # Skip only empty strings (pass flag with no value) + # Note: 0 and 0.0 are valid values, so use explicit check + if v != "" and v is not None: + if isinstance(v, str) and "\n" in v: + command += v.split("\n") + else: + command += [str(v)] # Add custom parameters for k, v in custom_params.items(): - if k in flag_params: - if isinstance(v, str): - is_enabled = v.lower() in {"true", "1", "yes", "on"} - else: - is_enabled = bool(v) - if is_enabled: - command += [f"-{k}"] - continue - if v == "" or v is None: - continue command += [f"-{k}"] - if isinstance(v, list): - command += [str(x) for x in v] - elif isinstance(v, bool): - command += [str(v).lower()] - else: - command += [str(v)] + # Skip only empty strings (pass flag with no value) + # Note: 0 and 0.0 are valid values, so use explicit check + if v != "" and v is not None: + if isinstance(v, list): + command += [str(x) for x in v] + else: + command += [str(v)] # Add threads parameter for TOPP tools command += ["-threads", str(threads_per_command)] commands.append(command) - for idx, cmd in enumerate(commands): - # Print list-form command joined into a single string for readability - print(f" πŸ”Ή Command {idx + 1}: {' '.join(cmd)}") - print("==========================================================\n") - + # check if a ini file has been written, if yes use it (contains custom defaults) + ini_path = Path(self.parameter_manager.ini_dir, tool + ".ini") + if ini_path.exists(): + command += ["-ini", str(ini_path)] # Run command(s) if len(commands) == 1: diff --git a/src/workflow/ParameterManager.py b/src/workflow/ParameterManager.py index 19e8700..b0c3626 100644 --- a/src/workflow/ParameterManager.py +++ b/src/workflow/ParameterManager.py @@ -129,30 +129,6 @@ def get_parameters_from_json(self) -> dict: except: st.error("**ERROR**: Attempting to load an invalid JSON parameter file. Reset to defaults.") return {} - - def get_merged_params(self, tool_instance_name: str, ini_params: dict = None) -> dict: - """ - Three-layer parameter merge: ini defaults < _defaults < user overrides. - - Args: - tool_instance_name: Instance name (or tool name) to look up in params.json. - ini_params: Base parameters from the .ini file. Optional β€” callers that - don't need the ini layer (e.g., run_topp, which passes -ini separately) - can omit this. - - Returns: - Merged dict with the effective value for each parameter. - """ - params = self.get_parameters_from_json() - defaults = params.get("_defaults", {}).get(tool_instance_name, {}) - user = params.get(tool_instance_name, {}) - - merged = {} - if ini_params: - merged.update(ini_params) - merged.update(defaults) - merged.update(user) - return merged def get_topp_parameters(self, tool: str) -> dict: """ diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index befde2e..0af89aa 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -613,12 +613,10 @@ def input_TOPP( num_cols: int = 4, exclude_parameters: List[str] = [], include_parameters: List[str] = [], - flag_parameters: List[str] = [], display_tool_name: bool = True, display_subsections: bool = True, display_subsection_tabs: bool = False, custom_defaults: dict = {}, - tool_instance_name: str = None, ) -> None: """ Generates input widgets for TOPP tool parameters dynamically based on the tool's @@ -630,59 +628,33 @@ def input_TOPP( num_cols (int, optional): Number of columns to use for the layout. Defaults to 3. exclude_parameters (List[str], optional): List of parameter names to exclude from the widget. Defaults to an empty list. include_parameters (List[str], optional): List of parameter names to include in the widget. Defaults to an empty list. - flag_parameters (List[str], optional): List of parameter names that should - be treated as no-value CLI flags during command construction. display_tool_name (bool, optional): Whether to display the TOPP tool name. Defaults to True. display_subsections (bool, optional): Whether to split parameters into subsections based on the prefix. Defaults to True. display_subsection_tabs (bool, optional): Whether to display main subsections in separate tabs (if more than one main section). Defaults to False. custom_defaults (dict, optional): Dictionary of custom defaults to use. Defaults to an empty dict. - tool_instance_name (str, optional): A unique instance name for this tool - invocation. Allows multiple instances of the same TOPP tool with - independent parameters (e.g., two IDFilter calls). If not provided, - defaults to topp_tool_name. The instance name is used for session - state keys and parameter storage, while topp_tool_name is used for - the actual tool executable and ini file creation. """ - # Default instance name to the tool name when not provided - if tool_instance_name is None: - tool_instance_name = topp_tool_name - - # Register instance-name β†’ real-tool-name mapping in session state - if "_topp_tool_instance_map" not in st.session_state: - st.session_state["_topp_tool_instance_map"] = {} - st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name - if "_topp_flag_params" not in st.session_state: - st.session_state["_topp_flag_params"] = {} - st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters) - # Persist flag metadata so execution still sees it outside UI reruns/session context. - params = self.parameter_manager.get_parameters_from_json() - if "_flag_params" not in params: - params["_flag_params"] = {} - params["_flag_params"][tool_instance_name] = list(flag_parameters) - with open(self.parameter_manager.params_file, "w", encoding="utf-8") as f: - json.dump(params, f, indent=4) if not display_subsections: display_subsection_tabs = False if display_subsection_tabs: display_subsections = True - # Create pristine ini file (never mutated with custom defaults) + # write defaults ini files ini_file_path = Path(self.parameter_manager.ini_dir, f"{topp_tool_name}.ini") + ini_existed = ini_file_path.exists() if not self.parameter_manager.create_ini(topp_tool_name): st.error(f"TOPP tool **'{topp_tool_name}'** not found.") return - - # Seed custom defaults into params.json under _defaults key - if custom_defaults: - params = self.parameter_manager.get_parameters_from_json() - if "_defaults" not in params: - params["_defaults"] = {} - params["_defaults"][tool_instance_name] = custom_defaults - with open(self.parameter_manager.params_file, "w", encoding="utf-8") as f: - json.dump(params, f, indent=4) - # Refresh self.params so widget resolution sees _defaults - self.params = self.parameter_manager.get_parameters_from_json() + if not ini_existed: + # update custom defaults if necessary + if custom_defaults: + param = poms.Param() + poms.ParamXMLFile().load(str(ini_file_path), param) + for key, value in custom_defaults.items(): + encoded_key = f"{topp_tool_name}:1:{key}".encode() + if encoded_key in param.keys(): + param.setValue(encoded_key, value) + poms.ParamXMLFile().store(str(ini_file_path), param) # read into Param object param = poms.Param() @@ -752,7 +724,6 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: ":".join(key.decode().split(":")[:-1]) ), } - p["is_flag"] = (b"flag" in param.getTags(key)) # Parameter sections and subsections as string (e.g. "section:subsection") if display_subsections: p["sections"] = ":".join( @@ -760,18 +731,18 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: ) params.append(p) - # Build ini_params dict for three-layer merge - ini_params = {} - for p in params: - name = p["key"].decode().split(":1:")[1] - ini_params[name] = p["value"] - - # Resolve effective values: ini < _defaults < user overrides - merged = self.parameter_manager.get_merged_params(tool_instance_name, ini_params=ini_params) + # for each parameter in params_decoded + # if a parameter with custom default value exists, use that value + # else check if the parameter is already in self.params, if yes take the value from self.params for p in params: name = p["key"].decode().split(":1:")[1] - if name in merged: - p["value"] = merged[name] + if topp_tool_name in self.params: + if name in self.params[topp_tool_name]: + p["value"] = self.params[topp_tool_name][name] + elif name in custom_defaults: + p["value"] = custom_defaults[name] + elif name in custom_defaults: + p["value"] = custom_defaults[name] # Ensure list parameters stay as lists after loading from JSON # (JSON may store single-item lists as strings) if p["original_is_list"] and isinstance(p["value"], str): @@ -805,7 +776,7 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: # Display tool name if required if display_tool_name: - st.markdown(f"**{tool_instance_name}**") + st.markdown(f"**{topp_tool_name}**") tab_names = [k for k in param_sections.keys() if ":" not in k] tabs = None @@ -833,53 +804,23 @@ def display_TOPP_params(params: dict, num_cols): cols = st.columns(num_cols) i = 0 for p in params: - # get key and name – use tool_instance_name in session state key - key_str = p['key'].decode() - if tool_instance_name != topp_tool_name: - key_str = key_str.replace(f"{topp_tool_name}:1:", f"{tool_instance_name}:1:", 1) - key = f"{self.parameter_manager.topp_param_prefix}{key_str}" + # get key and name + key = f"{self.parameter_manager.topp_param_prefix}{p['key'].decode()}" name = p["name"] try: # sometimes strings with newline, handle as list if isinstance(p["value"], str) and "\n" in p["value"]: p["value"] = p["value"].split("\n") - # no-value CLI flag parameters should be shown as checkboxes - if p.get("is_flag", False): - flag_default = p["value"] - if isinstance(flag_default, str): - flag_default = flag_default.lower() in {"true", "1", "yes", "on"} - else: - flag_default = bool(flag_default) - # Streamlit widget keys persist in session_state and can override - # updated custom_defaults. Normalize and seed key explicitly. - if key in st.session_state: - current = st.session_state[key] - if isinstance(current, str): - st.session_state[key] = current.lower() in {"true", "1", "yes", "on"} - else: - st.session_state[key] = bool(current) - else: - st.session_state[key] = flag_default - cols[i].selectbox( - name, - options=[True, False], - index=0 if st.session_state[key] else 1, - format_func=lambda x: "True" if x else "False", - help=p["description"], - key=key, - ) # bools - elif isinstance(p["value"], bool): - bool_value = ( - (p["value"] == "true") - if type(p["value"]) == str - else p["value"] - ) - cols[i].selectbox( + if isinstance(p["value"], bool): + cols[i].markdown("##") + cols[i].checkbox( name, - options=[True, False], - index=0 if bool_value else 1, - format_func=lambda x: "True" if x else "False", + value=( + (p["value"] == "true") + if type(p["value"]) == str + else p["value"] + ), help=p["description"], key=key, ) @@ -964,6 +905,7 @@ def on_multiselect_change(dk=display_key, tk=key): cols[i].error(f"Error in parameter **{p['name']}**.") print('Error parsing "' + p["name"] + '": ' + str(e)) + for section, params in param_sections.items(): if tabs is None: show_subsection_header(section, display_subsections) From b1dfb4d46bbb1635738a023adee560bb0c1c8115 Mon Sep 17 00:00:00 2001 From: jkvision1101 Date: Thu, 16 Jul 2026 13:27:30 +0900 Subject: [PATCH 02/10] Sync workflow files to upstream/main (StreamlitUI, ParameterManager, CommandExecutor) --- src/workflow/CommandExecutor.py | 40 ++--- src/workflow/ParameterManager.py | 142 +++++++++++++-- src/workflow/StreamlitUI.py | 293 ++++++++++++++++++++++++++++--- 3 files changed, 409 insertions(+), 66 deletions(-) diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 86f265b..11bb148 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -5,7 +5,7 @@ import threading from pathlib import Path from .Logger import Logger -from .ParameterManager import ParameterManager +from .ParameterManager import ParameterManager, bool_param_paths_from_param_xml_ini import sys import importlib.util import json @@ -216,7 +216,7 @@ def read_stderr(): stdout_thread.join() stderr_thread.join() - def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> bool: + def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool_instance_name: str = None) -> bool: """ Constructs and executes commands for the specified tool OpenMS TOPP tool based on the given input and output configurations. Ensures that all input/output file lists @@ -234,6 +234,9 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b tool (str): The executable name or path of the tool. input_output (dict): A dictionary specifying the input/output parameter names (as key) and their corresponding file paths (as value). custom_params (dict): A dictionary of custom parameters to pass to the tool. + tool_instance_name (str, optional): A unique instance name for this tool + invocation, used for parameter lookup when multiple instances of the + same tool exist. If not provided, defaults to the tool name. Returns: bool: True if all commands succeeded, False if any failed. @@ -242,6 +245,8 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b ValueError: If the lengths of input/output file lists are inconsistent, except for single string inputs. """ + # Use tool_instance_name for parameter lookup, fall back to tool name + params_key = tool_instance_name if tool_instance_name else tool # check input: any input lists must be same length, other items can be a single string # e.g. input_mzML : [list of n mzML files], output_featureXML : [list of n featureXML files], input_database : database.tsv io_lengths = [len(v) for v in input_output.values() if len(v) > 1] @@ -261,8 +266,8 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b commands = [] - # Load parameters for non-defaults - params = self.parameter_manager.get_parameters_from_json() + # Load merged parameters (_defaults + user overrides) for this tool instance + merged_params = self.parameter_manager.get_merged_params(params_key) # Construct commands for each process for i in range(n_processes): command = [tool] @@ -281,20 +286,20 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b # standard case, files was a list of strings, take the file name at index else: command += [value[i]] - # Add non-default TOPP tool parameters - if tool in params.keys(): - for k, v in params[tool].items(): - command += [f"-{k}"] - # Skip only empty strings (pass flag with no value) - # Note: 0 and 0.0 are valid values, so use explicit check - if v != "" and v is not None: - if isinstance(v, str) and "\n" in v: - command += v.split("\n") - else: - command += [str(v)] + # Add merged TOPP tool parameters (_defaults + user overrides) + for k, v in merged_params.items(): + command += [f"-{k}"] + # Skip only empty strings (pass flag with no value) + # Note: 0 and 0.0 are valid values, so use explicit check + if v != "" and v is not None: + if isinstance(v, str) and "\n" in v: + command += v.split("\n") + else: + command += [str(v)] # Add custom parameters for k, v in custom_params.items(): command += [f"-{k}"] + # Skip only empty strings (pass flag with no value) # Note: 0 and 0.0 are valid values, so use explicit check if v != "" and v is not None: @@ -306,11 +311,6 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b command += ["-threads", str(threads_per_command)] commands.append(command) - # check if a ini file has been written, if yes use it (contains custom defaults) - ini_path = Path(self.parameter_manager.ini_dir, tool + ".ini") - if ini_path.exists(): - command += ["-ini", str(ini_path)] - # Run command(s) if len(commands) == 1: return self.run_command(commands[0]) diff --git a/src/workflow/ParameterManager.py b/src/workflow/ParameterManager.py index b0c3626..2838c1b 100644 --- a/src/workflow/ParameterManager.py +++ b/src/workflow/ParameterManager.py @@ -3,8 +3,52 @@ import shutil import subprocess import streamlit as st +import xml.etree.ElementTree as ET from pathlib import Path + +def bool_param_paths_from_param_xml_ini(ini_path: Path, tool_stem: str) -> set[str]: + """ + Return short parameter paths for every ```` in a ParamXML .ini file. + + Paths match the suffix after ``Tool:1:`` in pyOpenMS (e.g. ``algorithm:epd:masstrace_snr_filtering``). + """ + try: + root = ET.parse(ini_path).getroot() + except (ET.ParseError, OSError): + return set() + + def local_tag(el: ET.Element) -> str: + t = el.tag + return t.rsplit("}", 1)[-1] if isinstance(t, str) and "}" in t else str(t) + + out: set[str] = set() + + def walk(el: ET.Element, parts: tuple[str, ...]) -> None: + for ch in el: + lt = local_tag(ch) + if lt == "NODE": + nm = ch.get("name") or "" + walk(ch, parts + (nm,)) + elif lt == "ITEM" and (ch.get("type") or "").lower() == "bool": + nm = ch.get("name") or "" + segs = [p for p in parts if p] + if nm: + segs.append(nm) + if not segs: + continue + # Strip tool root NODE name and instance NODE "1" (not part of pyOpenMS short keys) + while segs and segs[0] in (tool_stem, "1"): + segs.pop(0) + if segs: + out.add(":".join(segs)) + + for ch in root: + if local_tag(ch) == "NODE": + walk(ch, ()) + return out + + class ParameterManager: """ Manages the parameters for a workflow, including saving parameters to a JSON file, @@ -29,6 +73,29 @@ def __init__(self, workflow_dir: Path, workflow_name: str = None): # Store workflow name for preset loading; default to directory stem if not provided self.workflow_name = workflow_name or workflow_dir.stem + def bool_pairs_session_key(self) -> str: + """Session state key holding a set of (tool name, param path) for bool TOPP params.""" + return f"{self.ini_dir.parent.stem}-topp-bool-pairs" + + def get_bool_param_pairs(self) -> set: + """Return the cached set of (tool, param path) bool params; empty set if none.""" + return st.session_state.get(self.bool_pairs_session_key(), set()) + + def _merge_bool_params_from_ini(self, tool: str) -> None: + """Load tool.ini (XML) and merge type=bool parameter paths into session_state.""" + ini_path = Path(self.ini_dir, f"{tool}.ini") + if not ini_path.exists(): + return + try: + sk = self.bool_pairs_session_key() + if sk not in st.session_state: + st.session_state[sk] = set() + for short in bool_param_paths_from_param_xml_ini(ini_path, tool): + st.session_state[sk].add((tool, short)) + except RuntimeError: + # No Streamlit session (e.g. plain `python` import) + pass + def create_ini(self, tool: str) -> bool: """ Create an ini file for a TOPP tool if it doesn't exist. @@ -41,11 +108,14 @@ def create_ini(self, tool: str) -> bool: """ ini_path = Path(self.ini_dir, tool + ".ini") if ini_path.exists(): + self._merge_bool_params_from_ini(tool) return True try: subprocess.call([tool, "-write_ini", str(ini_path)]) except FileNotFoundError: return False + if ini_path.exists(): + self._merge_bool_params_from_ini(tool) return ini_path.exists() def save_parameters(self) -> None: @@ -65,7 +135,7 @@ def save_parameters(self) -> None: # Advanced parameters are only in session state if the view is active json_params = self.get_parameters_from_json() | json_params - # get a list of TOPP tools which are in session state + # get a list of TOPP tools (or tool instance names) which are in session state current_topp_tools = list( set( [ @@ -75,12 +145,16 @@ def save_parameters(self) -> None: ] ) ) - # for each TOPP tool, open the ini file + # Retrieve the instance-name β†’ real-tool-name mapping (set by input_TOPP) + tool_instance_map = st.session_state.get("_topp_tool_instance_map", {}) + # for each TOPP tool (or instance name), open the ini file for tool in current_topp_tools: - if not self.create_ini(tool): + # Resolve instance name to real tool name for create_ini / ini loading + real_tool = tool_instance_map.get(tool, tool) + if not self.create_ini(real_tool): # Could not create ini file - skip this tool continue - ini_path = Path(self.ini_dir, f"{tool}.ini") + ini_path = Path(self.ini_dir, f"{real_tool}.ini") if tool not in json_params: json_params[tool] = {} # load the param object @@ -92,19 +166,26 @@ def save_parameters(self) -> None: # Skip display keys used by multiselect widgets if key.endswith("_display"): continue - # get ini_key - ini_key = key.replace(self.topp_param_prefix, "").encode() + # get ini_key – map instance name back to real tool name + ini_key = key.replace(self.topp_param_prefix, "") + if tool != real_tool: + ini_key = ini_key.replace(f"{tool}:1:", f"{real_tool}:1:", 1) + ini_key = ini_key.encode() # get ini (default) value by ini_key ini_value = param.getValue(ini_key) is_list_param = isinstance(ini_value, list) - # check if value is different from default OR is an empty list parameter + # Effective default: _defaults value if present, else ini value + short_key = key.split(":1:")[1] + defaults = json_params.get("_defaults", {}).get(tool, {}) + default_value = defaults.get(short_key, ini_value) + # check if value is different from effective default OR is an empty list parameter if ( - (ini_value != value) - or (key.split(":1:")[1] in json_params[tool]) + (default_value != value) + or (short_key in json_params[tool]) or (is_list_param and not value) # Always save empty list params ): # store non-default value - json_params[tool][key.split(":1:")[1]] = value + json_params[tool][short_key] = value # Save to json file with open(self.params_file, "w", encoding="utf-8") as f: json.dump(json_params, f, indent=4) @@ -130,17 +211,44 @@ def get_parameters_from_json(self) -> dict: st.error("**ERROR**: Attempting to load an invalid JSON parameter file. Reset to defaults.") return {} - def get_topp_parameters(self, tool: str) -> dict: + def get_merged_params(self, tool_instance_name: str, ini_params: dict = None) -> dict: + """ + Three-layer parameter merge: ini defaults < _defaults < user overrides. + + Args: + tool_instance_name: Instance name (or tool name) to look up in params.json. + ini_params: Base parameters from the .ini file. Optional β€” callers that + don't need the ini layer (e.g., run_topp, which passes -ini separately) + can omit this. + + Returns: + Merged dict with the effective value for each parameter. + """ + params = self.get_parameters_from_json() + defaults = params.get("_defaults", {}).get(tool_instance_name, {}) + user = params.get(tool_instance_name, {}) + + merged = {} + if ini_params: + merged.update(ini_params) + merged.update(defaults) + merged.update(user) + return merged + + def get_topp_parameters(self, tool: str, tool_instance_name: str = None) -> dict: """ Get all parameters for a TOPP tool, merging defaults with user values. Args: - tool: Name of the TOPP tool (e.g., "CometAdapter") + tool: Name of the TOPP tool executable (e.g., "CometAdapter") + tool_instance_name: Optional instance name used for parameter storage + (e.g., "IDFilter_step1"). If not provided, defaults to tool name. Returns: Dict with parameter names as keys (without tool prefix) and their values. Returns empty dict if ini file doesn't exist. """ + instance_name = tool_instance_name or tool ini_path = Path(self.ini_dir, f"{tool}.ini") if not ini_path.exists(): return {} @@ -151,18 +259,14 @@ def get_topp_parameters(self, tool: str) -> dict: # Build dict from ini (extract short key names) prefix = f"{tool}:1:" - full_params = {} + ini_params = {} for key in param.keys(): key_str = key.decode() if isinstance(key, bytes) else str(key) if prefix in key_str: short_key = key_str.split(prefix, 1)[1] - full_params[short_key] = param.getValue(key) - - # Override with user-modified values from JSON - user_params = self.get_parameters_from_json().get(tool, {}) - full_params.update(user_params) + ini_params[short_key] = param.getValue(key) - return full_params + return self.get_merged_params(instance_name, ini_params=ini_params) def reset_to_default_parameters(self) -> None: """ diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 0af89aa..9c24dc2 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -23,6 +23,32 @@ from src.workflow._log_status import classify_log_outcome +def _mounted_data_root() -> Union[Path, None]: + """Return the validated mount root from the ``local_data_dir`` setting. + + The browser renders only when ``local_data_dir`` is an actual mount + point inside the container β€” i.e. the operator passed ``-v`` / + ``--bind`` / ``volumeMount`` to attach host data. Existence alone is + no longer sufficient because the image now pre-creates the path so + apptainer/singularity binds have a real attach target; without + ``os.path.ismount`` the browser would render an empty tree for every + user who didn't mount anything. + """ + settings = st.session_state.get("settings") or {} + raw = (settings.get("local_data_dir") or "").strip() + if not raw: + return None + try: + p = Path(raw).expanduser().resolve(strict=True) + except (OSError, RuntimeError): + return None + if not p.is_dir(): + return None + if not os.path.ismount(p): + return None + return p + + class StreamlitUI: """ Provides an interface for Streamlit applications to handle file uploads, @@ -77,6 +103,8 @@ def upload_widget( c1, c2 = st.columns(2) c1.markdown("**Upload file(s)**") + mount_root = _mounted_data_root() if st.session_state.location == "online" else None + if st.session_state.location == "local": c2_text, c2_checkbox = c2.columns([1.5, 1], gap="large") c2_text.markdown("**OR add files from local folder**") @@ -247,7 +275,19 @@ def upload_widget( "This means that the original files will be used instead. " ) - if fallback and not any([f for f in Path(files_dir).iterdir() if f.name != "external_files.txt"]): + if mount_root is not None: + with c2: + self._mounted_drive_browser(key, name, file_types, files_dir, mount_root) + + external_files_path = Path(files_dir, "external_files.txt") + has_real_files = any( + p.name != "external_files.txt" for p in files_dir.iterdir() + ) + has_external_picks = external_files_path.exists() and any( + line.strip() and os.path.exists(line.strip()) + for line in external_files_path.read_text().splitlines() + ) + if fallback and not has_real_files and not has_external_picks: if isinstance(fallback, str): fallback = [fallback] for f in fallback: @@ -303,6 +343,179 @@ def upload_widget( elif not fallback: st.warning(f"No **{name}** files!") + def _resolve_browser_cwd(self, key: str, mount_root: Path) -> Path: + """Read cwd for this widget from session state, confine it to mount_root.""" + sess_key = f"mounted_cwd_{key}" + raw = st.session_state.get(sess_key, str(mount_root)) + try: + cwd = Path(raw).expanduser().resolve(strict=True) + except (OSError, RuntimeError): + cwd = mount_root + if cwd != mount_root and mount_root not in cwd.parents: + cwd = mount_root + st.session_state[sess_key] = str(cwd) + return cwd + + def _mounted_drive_browser( + self, + key: str, + name: str, + file_types: List[str], + files_dir: Path, + mount_root: Path, + ) -> None: + """Render a tree browser for a mounted host directory. + + Selected files are referenced in place via ``external_files.txt`` β€” + the same mechanism the offline tkinter flow uses. + """ + external_files = Path(files_dir, "external_files.txt") + if not external_files.exists(): + external_files.touch() + + cwd = self._resolve_browser_cwd(key, mount_root) + sess_cwd_key = f"mounted_cwd_{key}" + + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + + with st.container(border=True): + st.markdown( + f"**Add {name} files from mounted directory** " + f"`{mount_root}`" + ) + + # Breadcrumbs: compact tertiary buttons separated by Β», + # with a right-aligned Parent button. + try: + rel = cwd.relative_to(mount_root) + segments = [mount_root.name] + list(rel.parts) if rel.parts else [mount_root.name] + except ValueError: + segments = [mount_root.name] + n = len(segments) + ratios: List[float] = [] + for i in range(n): + ratios.append(max(len(segments[i]), 3)) + if i < n - 1: + ratios.append(1) + ratios.append(20) # flexible spacer + ratios.append(6) # parent button slot + crumb_cols = st.columns(ratios, vertical_alignment="center") + col_idx = 0 + for i, seg in enumerate(segments): + target = mount_root.joinpath(*segments[1 : i + 1]) if i > 0 else mount_root + if crumb_cols[col_idx].button( + seg, + key=f"crumb_{key}_{i}", + type="tertiary", + ): + st.session_state[sess_cwd_key] = str(target) + st.rerun(scope="fragment") + col_idx += 1 + if i < n - 1: + crumb_cols[col_idx].markdown( + "Β»", + unsafe_allow_html=True, + ) + col_idx += 1 + # spacer column + col_idx += 1 + if cwd != mount_root: + if crumb_cols[col_idx].button( + "⬆ Parent", + key=f"mounted_parent_{key}", + type="tertiary", + ): + st.session_state[sess_cwd_key] = str(cwd.parent) + st.rerun(scope="fragment") + + try: + entries = sorted( + (p for p in cwd.iterdir() if not p.name.startswith(".")), + key=lambda p: (not p.is_dir(), p.name.lower()), + ) + except PermissionError: + st.error(f"Permission denied reading `{cwd}`.") + return + + def _is_match(p: Path) -> bool: + return any(p.name.endswith(f".{ft}") for ft in file_types) + + subdirs = [p for p in entries if p.is_dir() and not _is_match(p)] + bundled = [p for p in entries if p.is_dir() and _is_match(p)] + files = [p for p in entries if p.is_file() and _is_match(p)] + + for d in subdirs: + indent, body = st.columns([1, 60], vertical_alignment="center") + if body.button( + f"πŸ“‚ {d.name}/", + key=f"mounted_dir_{key}_{d.name}", + type="tertiary", + ): + st.session_state[sess_cwd_key] = str(d) + st.rerun(scope="fragment") + + selectable = bundled + files + selected_paths: List[str] = [] + for f in selectable: + cb_key = f"mounted_pick_{key}_{f}" + size_label = "" + if f.is_file(): + try: + size_mb = f.stat().st_size / (1024 * 1024) + size_label = f" Β· {size_mb:.1f} MB" + except OSError: + pass + icon = "πŸ—‚οΈ" if f.is_dir() else "πŸ“„" + if st.checkbox( + f"{icon} {f.name}{size_label}", + key=cb_key, + ): + selected_paths.append(str(f)) + + if not subdirs and not selectable: + st.info( + f"No subdirectories or files matching " + f"**{', '.join('.' + ft for ft in file_types)}** here." + ) + + count = len(selected_paths) + if st.button( + f"βž• Add {count} selected {name} file(s)" if count else f"βž• Add selected {name} file(s)", + key=f"mounted_add_{key}", + type="primary", + use_container_width=True, + disabled=count == 0, + ): + existing = set( + line.strip() + for line in external_files.read_text().splitlines() + if line.strip() + ) + added = 0 + with open(external_files, "a") as fh: + for p in selected_paths: + if p not in existing: + fh.write(f"{p}\n") + existing.add(p) + added += 1 + # Clear the checkboxes by removing their session keys. + for f in selectable: + st.session_state.pop(f"mounted_pick_{key}_{f}", None) + st.success(f"Added {added} file(s) from `{cwd}`.") + st.rerun(scope="fragment") + def select_input_file( self, key: str, @@ -617,6 +830,7 @@ def input_TOPP( display_subsections: bool = True, display_subsection_tabs: bool = False, custom_defaults: dict = {}, + tool_instance_name: str = None, ) -> None: """ Generates input widgets for TOPP tool parameters dynamically based on the tool's @@ -632,29 +846,43 @@ def input_TOPP( display_subsections (bool, optional): Whether to split parameters into subsections based on the prefix. Defaults to True. display_subsection_tabs (bool, optional): Whether to display main subsections in separate tabs (if more than one main section). Defaults to False. custom_defaults (dict, optional): Dictionary of custom defaults to use. Defaults to an empty dict. + tool_instance_name (str, optional): A unique instance name for this tool + invocation. Allows multiple instances of the same TOPP tool with + independent parameters (e.g., two IDFilter calls). If not provided, + defaults to topp_tool_name. The instance name is used for session + state keys and parameter storage, while topp_tool_name is used for + the actual tool executable and ini file creation. """ + # Default instance name to the tool name when not provided + if tool_instance_name is None: + tool_instance_name = topp_tool_name + + # Register instance-name β†’ real-tool-name mapping in session state + if "_topp_tool_instance_map" not in st.session_state: + st.session_state["_topp_tool_instance_map"] = {} + st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name if not display_subsections: display_subsection_tabs = False if display_subsection_tabs: display_subsections = True - # write defaults ini files + # Create pristine ini file (never mutated with custom defaults) ini_file_path = Path(self.parameter_manager.ini_dir, f"{topp_tool_name}.ini") - ini_existed = ini_file_path.exists() if not self.parameter_manager.create_ini(topp_tool_name): st.error(f"TOPP tool **'{topp_tool_name}'** not found.") return - if not ini_existed: - # update custom defaults if necessary - if custom_defaults: - param = poms.Param() - poms.ParamXMLFile().load(str(ini_file_path), param) - for key, value in custom_defaults.items(): - encoded_key = f"{topp_tool_name}:1:{key}".encode() - if encoded_key in param.keys(): - param.setValue(encoded_key, value) - poms.ParamXMLFile().store(str(ini_file_path), param) + + # Seed custom defaults into params.json under _defaults key + if custom_defaults: + params = self.parameter_manager.get_parameters_from_json() + if "_defaults" not in params: + params["_defaults"] = {} + params["_defaults"][tool_instance_name] = custom_defaults + with open(self.parameter_manager.params_file, "w", encoding="utf-8") as f: + json.dump(params, f, indent=4) + # Refresh self.params so widget resolution sees _defaults + self.params = self.parameter_manager.get_parameters_from_json() # read into Param object param = poms.Param() @@ -731,18 +959,18 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: ) params.append(p) - # for each parameter in params_decoded - # if a parameter with custom default value exists, use that value - # else check if the parameter is already in self.params, if yes take the value from self.params + # Build ini_params dict for three-layer merge + ini_params = {} for p in params: name = p["key"].decode().split(":1:")[1] - if topp_tool_name in self.params: - if name in self.params[topp_tool_name]: - p["value"] = self.params[topp_tool_name][name] - elif name in custom_defaults: - p["value"] = custom_defaults[name] - elif name in custom_defaults: - p["value"] = custom_defaults[name] + ini_params[name] = p["value"] + + # Resolve effective values: ini < _defaults < user overrides + merged = self.parameter_manager.get_merged_params(tool_instance_name, ini_params=ini_params) + for p in params: + name = p["key"].decode().split(":1:")[1] + if name in merged: + p["value"] = merged[name] # Ensure list parameters stay as lists after loading from JSON # (JSON may store single-item lists as strings) if p["original_is_list"] and isinstance(p["value"], str): @@ -776,7 +1004,7 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: # Display tool name if required if display_tool_name: - st.markdown(f"**{topp_tool_name}**") + st.markdown(f"**{tool_instance_name}**") tab_names = [k for k in param_sections.keys() if ":" not in k] tabs = None @@ -804,8 +1032,11 @@ def display_TOPP_params(params: dict, num_cols): cols = st.columns(num_cols) i = 0 for p in params: - # get key and name - key = f"{self.parameter_manager.topp_param_prefix}{p['key'].decode()}" + # get key and name – use tool_instance_name in session state key + key_str = p['key'].decode() + if tool_instance_name != topp_tool_name: + key_str = key_str.replace(f"{topp_tool_name}:1:", f"{tool_instance_name}:1:", 1) + key = f"{self.parameter_manager.topp_param_prefix}{key_str}" name = p["name"] try: # sometimes strings with newline, handle as list @@ -1377,7 +1608,8 @@ def remove_full_paths(d: dict) -> dict: general = {} for k, v in params.items(): - # skip if v is a file path + if k == "_defaults": + continue if isinstance(v, dict): topp[k] = v elif ".py" in k: @@ -1388,6 +1620,13 @@ def remove_full_paths(d: dict) -> dict: else: general[k] = v + # Merge _defaults into topp so summary shows custom defaults + user overrides + defaults = params.get("_defaults", {}) + for tool_name, default_vals in defaults.items(): + if tool_name not in topp: + topp[tool_name] = {} + topp[tool_name] = {**default_vals, **topp.get(tool_name, {})} + markdown = [] def dict_to_markdown(d: dict): From c12bc1cfbcd2d158f9d2984cbdbe435dbbbc421b Mon Sep 17 00:00:00 2001 From: Yoo HoJun Date: Wed, 15 Jul 2026 15:04:48 +0900 Subject: [PATCH 03/10] Add support for boolean CLI flag parameters in TOPP tools Allows input_TOPP() to designate parameters as flags (present/absent) instead of key-value pairs, persisted to params.json and session_state so run_topp() can correctly build the command line. --- src/workflow/CommandExecutor.py | 53 +++++++++++++++++++++++---------- src/workflow/StreamlitUI.py | 13 ++++++++ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 11bb148..042c5e1 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -268,6 +268,14 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool # Load merged parameters (_defaults + user overrides) for this tool instance merged_params = self.parameter_manager.get_merged_params(params_key) + + # Load flag parameter names: params.json takes priority (survives session restart), + # session_state is the live fallback during the current session. + flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {}) + if not flag_map: + flag_map = st.session_state.get("_topp_flag_params", {}) + flag_params: set = set(flag_map.get(params_key, [])) + # Construct commands for each process for i in range(n_processes): command = [tool] @@ -288,25 +296,40 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool command += [value[i]] # Add merged TOPP tool parameters (_defaults + user overrides) for k, v in merged_params.items(): - command += [f"-{k}"] - # Skip only empty strings (pass flag with no value) - # Note: 0 and 0.0 are valid values, so use explicit check - if v != "" and v is not None: - if isinstance(v, str) and "\n" in v: - command += v.split("\n") + if k in flag_params: + # CLI flag: include "-k" only when truthy, omit when false + if isinstance(v, str): + is_enabled = v.lower() == "true" else: - command += [str(v)] + is_enabled = bool(v) + if is_enabled: + command += [f"-{k}"] + continue + # Regular parameter: skip empty/None, append value otherwise + if v == "" or v is None: + continue + command += [f"-{k}"] + if isinstance(v, str) and "\n" in v: + command += v.split("\n") + else: + command += [str(v)] # Add custom parameters for k, v in custom_params.items(): - command += [f"-{k}"] - - # Skip only empty strings (pass flag with no value) - # Note: 0 and 0.0 are valid values, so use explicit check - if v != "" and v is not None: - if isinstance(v, list): - command += [str(x) for x in v] + if k in flag_params: + if isinstance(v, str): + is_enabled = v.lower() == "true" else: - command += [str(v)] + is_enabled = bool(v) + if is_enabled: + command += [f"-{k}"] + continue + if v == "" or v is None: + continue + command += [f"-{k}"] + if isinstance(v, list): + command += [str(x) for x in v] + else: + command += [str(v)] # Add threads parameter for TOPP tools command += ["-threads", str(threads_per_command)] commands.append(command) diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 9c24dc2..4bd53b1 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -826,6 +826,7 @@ def input_TOPP( num_cols: int = 4, exclude_parameters: List[str] = [], include_parameters: List[str] = [], + flag_parameters: List[str] = [], display_tool_name: bool = True, display_subsections: bool = True, display_subsection_tabs: bool = False, @@ -862,6 +863,18 @@ def input_TOPP( st.session_state["_topp_tool_instance_map"] = {} st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name + # Persist flag_parameters to session_state and params.json so run_topp + # can skip appending a value for these boolean CLI flags. + if "_topp_flag_params" not in st.session_state: + st.session_state["_topp_flag_params"] = {} + st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters) + _fp = self.parameter_manager.get_parameters_from_json() + if "_flag_params" not in _fp: + _fp["_flag_params"] = {} + _fp["_flag_params"][tool_instance_name] = list(flag_parameters) + with open(self.parameter_manager.params_file, "w", encoding="utf-8") as _f: + json.dump(_fp, _f, indent=4) + if not display_subsections: display_subsection_tabs = False if display_subsection_tabs: From af57d216085cdae53af46d02cd3b44979c7ac09f Mon Sep 17 00:00:00 2001 From: jkvision1101 Date: Thu, 16 Jul 2026 14:45:41 +0900 Subject: [PATCH 04/10] InputTOPP: add reactive option and split impl/fragment for conditional UI --- pr-397.patch | 119 ++++++++++++++++++++++++++++++++++++ src/WorkflowTest.py | 8 ++- src/workflow/StreamlitUI.py | 74 +++++++++++++++++++++- 3 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 pr-397.patch diff --git a/pr-397.patch b/pr-397.patch new file mode 100644 index 0000000..bb97819 --- /dev/null +++ b/pr-397.patch @@ -0,0 +1,119 @@ +From 268348f6da9809750e3116535f4f3fe9defc7ab1 Mon Sep 17 00:00:00 2001 +From: Yoo HoJun +Date: Wed, 15 Jul 2026 15:04:48 +0900 +Subject: [PATCH] Add support for boolean CLI flag parameters in TOPP tools + +Allows input_TOPP() to designate parameters as flags (present/absent) +instead of key-value pairs, persisted to params.json and session_state +so run_topp() can correctly build the command line. +--- + src/workflow/CommandExecutor.py | 53 +++++++++++++++++++++++---------- + src/workflow/StreamlitUI.py | 13 ++++++++ + 2 files changed, 51 insertions(+), 15 deletions(-) + +diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py +index 11bb1486..042c5e11 100644 +--- a/src/workflow/CommandExecutor.py ++++ b/src/workflow/CommandExecutor.py +@@ -268,6 +268,14 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool + + # Load merged parameters (_defaults + user overrides) for this tool instance + merged_params = self.parameter_manager.get_merged_params(params_key) ++ ++ # Load flag parameter names: params.json takes priority (survives session restart), ++ # session_state is the live fallback during the current session. ++ flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {}) ++ if not flag_map: ++ flag_map = st.session_state.get("_topp_flag_params", {}) ++ flag_params: set = set(flag_map.get(params_key, [])) ++ + # Construct commands for each process + for i in range(n_processes): + command = [tool] +@@ -288,25 +296,40 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool + command += [value[i]] + # Add merged TOPP tool parameters (_defaults + user overrides) + for k, v in merged_params.items(): +- command += [f"-{k}"] +- # Skip only empty strings (pass flag with no value) +- # Note: 0 and 0.0 are valid values, so use explicit check +- if v != "" and v is not None: +- if isinstance(v, str) and "\n" in v: +- command += v.split("\n") ++ if k in flag_params: ++ # CLI flag: include "-k" only when truthy, omit when false ++ if isinstance(v, str): ++ is_enabled = v.lower() == "true" + else: +- command += [str(v)] ++ is_enabled = bool(v) ++ if is_enabled: ++ command += [f"-{k}"] ++ continue ++ # Regular parameter: skip empty/None, append value otherwise ++ if v == "" or v is None: ++ continue ++ command += [f"-{k}"] ++ if isinstance(v, str) and "\n" in v: ++ command += v.split("\n") ++ else: ++ command += [str(v)] + # Add custom parameters + for k, v in custom_params.items(): +- command += [f"-{k}"] +- +- # Skip only empty strings (pass flag with no value) +- # Note: 0 and 0.0 are valid values, so use explicit check +- if v != "" and v is not None: +- if isinstance(v, list): +- command += [str(x) for x in v] ++ if k in flag_params: ++ if isinstance(v, str): ++ is_enabled = v.lower() == "true" + else: +- command += [str(v)] ++ is_enabled = bool(v) ++ if is_enabled: ++ command += [f"-{k}"] ++ continue ++ if v == "" or v is None: ++ continue ++ command += [f"-{k}"] ++ if isinstance(v, list): ++ command += [str(x) for x in v] ++ else: ++ command += [str(v)] + # Add threads parameter for TOPP tools + command += ["-threads", str(threads_per_command)] + commands.append(command) +diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py +index 9c24dc2c..4bd53b18 100644 +--- a/src/workflow/StreamlitUI.py ++++ b/src/workflow/StreamlitUI.py +@@ -826,6 +826,7 @@ def input_TOPP( + num_cols: int = 4, + exclude_parameters: List[str] = [], + include_parameters: List[str] = [], ++ flag_parameters: List[str] = [], + display_tool_name: bool = True, + display_subsections: bool = True, + display_subsection_tabs: bool = False, +@@ -862,6 +863,18 @@ def input_TOPP( + st.session_state["_topp_tool_instance_map"] = {} + st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name + ++ # Persist flag_parameters to session_state and params.json so run_topp ++ # can skip appending a value for these boolean CLI flags. ++ if "_topp_flag_params" not in st.session_state: ++ st.session_state["_topp_flag_params"] = {} ++ st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters) ++ _fp = self.parameter_manager.get_parameters_from_json() ++ if "_flag_params" not in _fp: ++ _fp["_flag_params"] = {} ++ _fp["_flag_params"][tool_instance_name] = list(flag_parameters) ++ with open(self.parameter_manager.params_file, "w", encoding="utf-8") as _f: ++ json.dump(_fp, _f, indent=4) ++ + if not display_subsections: + display_subsection_tabs = False + if display_subsection_tabs: diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index 4abbf92..2af9bda 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -2,8 +2,8 @@ from pathlib import Path import pandas as pd import plotly.express as px -from streamlit_plotly_events import plotly_events -from pyopenms import IdXMLFile +#from streamlit_plotly_events import plotly_events +#from pyopenms import IdXMLFile from scipy.stats import ttest_ind import numpy as np import mygene @@ -14,7 +14,7 @@ from src.common.common import page_setup from src.common.results_helpers import get_abundance_data from src.common.results_helpers import parse_idxml, build_spectra_cache -from openms_insight import Table, Heatmap, LinePlot, SequenceView +#from openms_insight import Table, Heatmap, LinePlot, SequenceView # params = page_setup() class WorkflowTest(WorkflowManager): @@ -129,6 +129,7 @@ def configure(self) -> None: "PeptideIndexing:unmatched_action": "warn", "PeptideIndexing:decoy_string": "rev_", }, + flag_parameters=["PeptideIndexing:IL_equivalent"], include_parameters=comet_include, exclude_parameters=["second_enzyme"], ) @@ -152,6 +153,7 @@ def configure(self) -> None: "score_type": "pep", "post_processing_tdc": "true", }, + flag_parameters=["post_processing_tdc"], include_parameters=percolator_include, exclude_parameters=["out_type"], ) diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 4bd53b1..77ca01d 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -819,7 +819,6 @@ def format_files(input: Any) -> List[str]: self.parameter_manager.save_parameters() - @st.fragment def input_TOPP( self, topp_tool_name: str, @@ -832,6 +831,79 @@ def input_TOPP( display_subsection_tabs: bool = False, custom_defaults: dict = {}, tool_instance_name: str = None, + reactive: bool = False, + ) -> None: + """ + Wrapper for TOPP parameter input. When `reactive` is True the + implementation is rendered directly in the parent context so changes + trigger a parent re-render; otherwise the widgets are rendered inside + a `st.fragment` to isolate reruns for performance. + """ + if reactive: + return self._input_TOPP_impl( + topp_tool_name=topp_tool_name, + num_cols=num_cols, + exclude_parameters=exclude_parameters, + include_parameters=include_parameters, + flag_parameters=flag_parameters, + display_tool_name=display_tool_name, + display_subsections=display_subsections, + display_subsection_tabs=display_subsection_tabs, + custom_defaults=custom_defaults, + tool_instance_name=tool_instance_name, + ) + return self._input_TOPP_fragmented( + topp_tool_name=topp_tool_name, + num_cols=num_cols, + exclude_parameters=exclude_parameters, + include_parameters=include_parameters, + flag_parameters=flag_parameters, + display_tool_name=display_tool_name, + display_subsections=display_subsections, + display_subsection_tabs=display_subsection_tabs, + custom_defaults=custom_defaults, + tool_instance_name=tool_instance_name, + ) + + @st.fragment + def _input_TOPP_fragmented( + self, + topp_tool_name: str, + num_cols: int = 4, + exclude_parameters: List[str] = [], + include_parameters: List[str] = [], + flag_parameters: List[str] = [], + display_tool_name: bool = True, + display_subsections: bool = True, + display_subsection_tabs: bool = False, + custom_defaults: dict = {}, + tool_instance_name: str = None, + ) -> None: + return self._input_TOPP_impl( + topp_tool_name=topp_tool_name, + num_cols=num_cols, + exclude_parameters=exclude_parameters, + include_parameters=include_parameters, + flag_parameters=flag_parameters, + display_tool_name=display_tool_name, + display_subsections=display_subsections, + display_subsection_tabs=display_subsection_tabs, + custom_defaults=custom_defaults, + tool_instance_name=tool_instance_name, + ) + + def _input_TOPP_impl( + self, + topp_tool_name: str, + num_cols: int = 4, + exclude_parameters: List[str] = [], + include_parameters: List[str] = [], + flag_parameters: List[str] = [], + display_tool_name: bool = True, + display_subsections: bool = True, + display_subsection_tabs: bool = False, + custom_defaults: dict = {}, + tool_instance_name: str = None, ) -> None: """ Generates input widgets for TOPP tool parameters dynamically based on the tool's From 7c5a2250d0e79ae796adcea87c4950b63a0ed12b Mon Sep 17 00:00:00 2001 From: Yoo HoJun Date: Thu, 16 Jul 2026 15:15:14 +0900 Subject: [PATCH 05/10] Add LFQ/TMT workflow mode split and downstream analysis pages Split WorkflowTest configure() into separate LFQ and TMT tool tabs, and add new preprocessing/analysis pages (filtering, normalization, imputation, statistical testing, GO enrichment, clustered heatmap, pathway analysis) built on openms_insight engine functions. --- content/enrichment.py | 141 ++ content/filtering.py | 173 +++ content/imputation.py | 145 ++ content/normalization.py | 242 ++++ content/results_abundance.py | 138 +- content/results_heatmap.py | 79 +- content/results_heatmap_clustered.py | 107 ++ content/results_pathway_analysis.py | 258 ++++ content/results_pca.py | 177 ++- content/results_proteomicslfq.py | 5 +- content/results_volcano.py | 91 +- content/statistical.py | 165 +++ src/WorkflowTest.py | 1841 +++++++++++++++++--------- src/common/results_helpers.py | 287 ++-- 14 files changed, 2971 insertions(+), 878 deletions(-) create mode 100644 content/enrichment.py create mode 100644 content/filtering.py create mode 100644 content/imputation.py create mode 100644 content/normalization.py create mode 100644 content/results_heatmap_clustered.py create mode 100644 content/results_pathway_analysis.py create mode 100644 content/statistical.py diff --git a/content/enrichment.py b/content/enrichment.py new file mode 100644 index 0000000..62fc9fa --- /dev/null +++ b/content/enrichment.py @@ -0,0 +1,141 @@ +"""Pathway Analysis Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column +# Import GO Enrichment modules from openms_insight engine +from openms_insight.analysis.enrichment import calculate_go_enrichment + +params = page_setup() +st.title("GO Enrichment Analysis") + +st.markdown( + """ +Identify overrepresented biological themes (BP, CC, MF) within your differentially expressed protein features using MyGene.info and Fisher's Exact Test. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# --- STEP 1: Upstream Statistics Checkpoint --- +if ( + "statistics_df" in st.session_state + and st.session_state["statistics_df"] is not None +): + final_statistics_report = st.session_state["statistics_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using analyzed matrices from the **Statistical Inference** step." + ) +else: + st.warning( + "⚠️ **Missing Prerequisites**: Statistical inference data not detected. Please run hypothesis testing first." + ) + st.page_link( + "content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬" + ) + st.stop() + +# --- STEP 2: Preprocessing Mapping Key Configuration --- +# Identify target identifier columns dynamically +abundance_result = get_abundance_data(st.session_state["workspace"]) +id_col = get_id_column(st.session_state["workspace"], abundance_result[0]) if abundance_result else "ProteinName" +if id_col not in final_statistics_report.columns: + st.error(f"❌ Structural Error: Column '{id_col}' is missing from the active matrix context.") + st.stop() + +# --- SECTION 1: Parameter Setup & Dynamic Cutoff Labels --- +st.subheader("Configure Enrichment Thresholds") + +# Check if target p-value should be adjusted or raw based on previous selections (Fallback safely to 'p-adj') +target_p_col = "p-adj" if "p-adj" in final_statistics_report.columns else "p-value" +p_label = ( + "Adjusted P-value (p-adj) Cutoff" + if target_p_col == "p-adj" + else "Raw P-value (p-value) Cutoff" +) + +ui_go_col1, ui_go_col2 = st.columns(2) + +with ui_go_col1: + p_cutoff = st.number_input( + f"πŸ”¬ {p_label}", + min_value=0.0001, + max_value=1.0, + value=0.05, + step=0.01, + format="%.4f", + help="Proteins with significance metrics below this value are mapped to the foreground cohort.", + ) + +with ui_go_col2: + fc_cutoff = st.number_input( + "πŸ“ˆ Absolute Difference Cutoff (|log2FC|)", + min_value=0.0, + max_value=10.0, + value=1.0, + step=0.1, + format="%.2f", + help="Proteins with absolute log2 fold change greater than or equal to this threshold will be selected.", + ) + +# --- SECTION 2: Execution and Interactive View Charts --- +st.markdown("
", unsafe_allow_html=True) +if st.button("πŸš€ Run GO Enrichment Analysis", type="primary", key="run_go_analysis"): + + with st.spinner("Querying MyGene.info API & executing hyper-geometric calculation loops..."): + # Convert internal pandas DataFrame to openms_insight Polars DataFrame expectation + stats_pl = pl.from_pandas(final_statistics_report) + + status, output = calculate_go_enrichment( + final_report=stats_pl, + id_col=id_col, + target_p_col=target_p_col, + p_cutoff=p_cutoff, + fc_cutoff=fc_cutoff, + ) + + # Route response structures based on analysis output status code + if status == "empty_data": + st.error("❌ No valid statistical rows found containing standard columns to run GO alignment.") + + elif status == "insufficient_proteins": + st.warning( + f"⚠️ Not enough significant proteins found to construct target datasets. " + f"(Criteria: {target_p_col} < {p_cutoff:.4f}, |log2FC| β‰₯ {fc_cutoff:.2f})." + ) + st.info(f"πŸ’‘ Found significant proteins count: **{output}**. Try relaxing your p-value or log2FC filters.") + + elif status == "success": + st.success("β­• GO Enrichment Analysis completed successfully!") + + # Display operational matrix scale + st.markdown( + f"πŸ“Š **Analysis Profile Scope**: Mapped **{output['fg_count']}** significant foreground profiles out of **{output['bg_count']}** reference background items." + ) + + # Build multi-tab interface layer for ontology subcategories + tabs = st.tabs([ + "🧬 Biological Process (BP)", + "πŸ”¬ Cellular Component (CC)", + "πŸ§ͺ Molecular Function (MF)" + ]) + categories_data = output["categories"] + + for idx, go_type in enumerate(["BP", "CC", "MF"]): + with tabs[idx]: + fig = categories_data[go_type]["fig"] + df_go = categories_data[go_type]["df"] + + if fig is not None and df_go is not None: + # Render plotly bar figures generated straight from backend engine + st.plotly_chart(fig, use_container_width=True) + + st.subheader(f"πŸ“Š {go_type} Results Dataframe") + st.dataframe(df_go, use_container_width=True) + else: + st.info(f"No statistically overrepresented terms identified for Category: **{go_type}**") \ No newline at end of file diff --git a/content/filtering.py b/content/filtering.py new file mode 100644 index 0000000..2bad00c --- /dev/null +++ b/content/filtering.py @@ -0,0 +1,173 @@ +"""Filtering Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import filtering functions from openms_insight package +from openms_insight.analysis.filter import ( + filter_low_abundance, + filter_low_repeatability, + filter_low_variance, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Filtering") + +st.markdown( + """ +Filter out low-quality proteins from your dataset based on abundance, repeatability, or variance thresholds. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Identify actual sample columns dynamically +sample_cols = [ + c + for c in pivot_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Original Data View --- +st.subheader("Original Abundance Table") +st.markdown( + f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering." +) +st.dataframe(pivot_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Filter Configuration --- +st.subheader("Configure Filter Engine") + +# Prepare Polars Metadata DataFrame required by openms_insight functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for filtering strategy +filter_method = st.selectbox( + "Select Filtering Method", + options=["Low Abundance", "Low Repeatability", "Low Variance"], + index=0, + help="Choose the statistical criteria to prune unreliable protein entries.", +) + +# Render threshold sliders dynamically based on the selected filter method +if filter_method == "Low Abundance": + st.markdown( + "**Low Abundance Filter**: Keeps rows where at least one group's median is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +elif filter_method == "Low Repeatability": + st.markdown( + "**Low Repeatability Filter**: Keeps rows where at least one group has a missing value ratio within the allowed maximum." + ) + threshold = st.slider( + "Max Missing Ratio", + min_value=0.0, + max_value=100.0, + value=50.0, + step=5.0, + help="Allowed missing value (zero or null) ratio per group.", + ) + +elif filter_method == "Low Variance": + st.markdown( + "**Low Variance Filter**: Keeps rows where at least one group's variance is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +# --- SECTION 3: Filter Execution and Collected Results View --- +if st.button("Apply Filter", type="primary"): + # Convert the original Pandas DataFrame into a Polars LazyFrame graph + quant_lazy = pl.from_pandas(pivot_df).lazy() + + # Route execution to the chosen openms_insight engine function + if filter_method == "Low Abundance": + filtered_lazy = filter_low_abundance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + elif filter_method == "Low Repeatability": + # Convert percent slider input to ratio expected by the function (e.g., 50.0% -> 0.5) + filtered_lazy = filter_low_repeatability( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + max_missing_ratio=threshold / 100.0, + ) + elif filter_method == "Low Variance": + filtered_lazy = filter_low_variance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + + # Collect the evaluated lazy graph and convert back to Pandas for visualization + filtered_df = strip_stat_columns(filtered_lazy.collect().to_pandas()) + st.session_state["filtered_df"] = filtered_df + + # Layout response metrics and the filtered matrix + st.success(f"Successfully applied **{filter_method}** filter!") + + # Display dataset scale compression stats + col1, col2, col3 = st.columns(3) + col1.metric("Original Proteins", pivot_df.shape[0]) + col2.metric("Filtered Proteins", filtered_df.shape[0]) + col3.metric( + "Removed Proteins", pivot_df.shape[0] - filtered_df.shape[0], delta=None + ) + + st.subheader("Filtered Abundance Table") + if filtered_df.empty: + st.warning( + "The filtered table is empty. Try relaxing the threshold constraints." + ) + else: + st.dataframe(filtered_df, use_container_width=True) \ No newline at end of file diff --git a/content/imputation.py b/content/imputation.py new file mode 100644 index 0000000..4350263 --- /dev/null +++ b/content/imputation.py @@ -0,0 +1,145 @@ +"""Imputation Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import imputation algorithms from openms_insight engine +from openms_insight.analysis.imputation import impute_mar, impute_smallest_value + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Missing Value Imputation") + +st.markdown( + """ +Handle missing values (zeros or nulls) in your quantification matrix using biological group-aware (MAR) or absolute lowest limit (MNAR) techniques. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load base dataset and clean dictionary keys +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Pipeline Checkpoint: Fetch upstream filtered data if available, fallback to raw pivot matrix +if "filtered_df" in st.session_state and st.session_state["filtered_df"] is not None: + base_df = strip_stat_columns(st.session_state["filtered_df"]) + st.session_state["filtered_df"] = base_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No filtering history found. Operating on the original unfiltered table." + ) + +# 2. Identify actual sample columns dynamically based on the current active matrix +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Input Matrix Summary --- +st.subheader("Input Matrix Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Imputation Configuration --- +st.subheader("Configure Imputation Engine") + +# Build Polars structural metadata DataFrame +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for core missingness assumption strategy +impute_category = st.selectbox( + "Select Imputation Class", + options=["MAR (Missing At Random)", "MNAR (Missing Not At Random)"], + index=0, + help="MAR uses group metrics (Mean/Median). MNAR shifts values below the limit of detection.", +) + +# Render algorithmic options sub-menus based on the parent selection +if impute_category == "MAR (Missing At Random)": + st.markdown( + "**Group Character Imputation**: Fills missing metrics leveraging sample properties belonging to the same group." + ) + strategy_opt = st.radio( + "Mathematical Strategy", + options=["median", "mean"], + index=0, + horizontal=True, + ) + +elif impute_category == "MNAR (Missing Not At Random)": + st.markdown( + "**Smallest Value Imputation**: Replaces missing items with the minimum values detected to reflect technical dropout limits." + ) + scope_opt = st.radio( + "Detection Minimum Scope", + options=["row", "global"], + index=0, + horizontal=True, + help="'row' targets current protein minimum; 'global' searches the entire mass spectrometry matrix profile.", + ) + +# --- SECTION 3: Imputation Execution --- +if st.button("Apply Imputation", type="primary"): + # Initialize optimization pipeline graph via lazy loading conversion + quant_lazy = pl.from_pandas(base_df).lazy() + + # Route configuration matrix parameters to designated engine function channels + if impute_category == "MAR (Missing At Random)": + imputed_lazy = impute_mar( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + strategy=strategy_opt, + ) + elif impute_category == "MNAR (Missing Not At Random)": + imputed_lazy = impute_smallest_value( + quantification_data=quant_lazy, metadata=metadata_pl, scope=scope_opt + ) + + # Resolve lazy graph optimization tree and push to display data frame structure + imputed_df = strip_stat_columns(imputed_lazy.collect().to_pandas()) + + # πŸ’Ύ Save current output into Session State for down-stream processing (Normalization, Statistics) + st.session_state["imputed_df"] = imputed_df + + st.success(f"Successfully finalized **{impute_category}** imputation step!") + + # Calculate and display a quick performance matrix check + st.subheader("Imputed Result Table") + st.dataframe(imputed_df, use_container_width=True) \ No newline at end of file diff --git a/content/normalization.py b/content/normalization.py new file mode 100644 index 0000000..c0e97e8 --- /dev/null +++ b/content/normalization.py @@ -0,0 +1,242 @@ +"""Normalization Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import normalization engine functions from openms_insight +from openms_insight.analysis.normalization import ( + normalize_samples, + scale_data, + transform_data, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None: + """Keep preprocessing tables intensity-only before statistical analysis.""" + if df is None: + return None + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Normalization & Scaling") + +st.markdown( + """ +Standardize and transform your protein abundance profiles to correct for technical variations and optimize statistical distributions. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +filtered_df = strip_stat_columns(st.session_state.get("filtered_df")) +imputed_df = strip_stat_columns(st.session_state.get("imputed_df")) +normalized_df = strip_stat_columns(st.session_state.get("normalized_df")) +if filtered_df is not None: + st.session_state["filtered_df"] = filtered_df +if imputed_df is not None: + st.session_state["imputed_df"] = imputed_df +if normalized_df is not None: + st.session_state["normalized_df"] = normalized_df + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = imputed_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = filtered_df + st.warning( + "⚠️ **Imputation Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original unfiltered table." + ) + +# 2. Extract actual active sample columns dynamically +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("### Pipeline Overview") +st.caption("Data flows in order: Filtering -> Imputation -> Normalization") + +step_rows = [ + { + "Step": "Filtering", + "Status": "Done" if filtered_df is not None else "Not run", + "Rows": filtered_df.shape[0] if filtered_df is not None else "-", + "Cols": filtered_df.shape[1] if filtered_df is not None else "-", + }, + { + "Step": "Imputation", + "Status": "Done" if imputed_df is not None else "Not run", + "Rows": imputed_df.shape[0] if imputed_df is not None else "-", + "Cols": imputed_df.shape[1] if imputed_df is not None else "-", + }, + { + "Step": "Normalization", + "Status": "Done" if normalized_df is not None else "Not run", + "Rows": normalized_df.shape[0] if normalized_df is not None else "-", + "Cols": normalized_df.shape[1] if normalized_df is not None else "-", + }, +] +st.dataframe(pd.DataFrame(step_rows), hide_index=True, use_container_width=True) + +with st.expander("Show step tables", expanded=False): + if filtered_df is not None: + st.markdown("#### Filtering output") + st.dataframe(filtered_df.head(10), use_container_width=True) + if imputed_df is not None: + st.markdown("#### Imputation output") + st.dataframe(imputed_df.head(10), use_container_width=True) + if normalized_df is not None: + st.markdown("#### Normalization output") + st.dataframe(normalized_df.head(10), use_container_width=True) + if filtered_df is None and imputed_df is None and normalized_df is None: + st.info("No preprocessing outputs yet. Start from Filtering.") + +st.markdown("---") + +# --- SECTION 2: Normalization Parameter Configuration --- +st.subheader("Configure Preprocessing & Scaling Chains") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2, col3 = st.columns(3) + +with col1: + st.markdown("### 🧬 1. Mathematical Transformation") + transform_strategy = st.selectbox( + "Select Transformation", + options=["None", "log2", "log10", "square_root", "cube_root"], + index=0, + help="Compress data dynamic range and stabilize heteroscedastic variance profiles.", + ) + +with col2: + st.markdown("### πŸ§ͺ 2. Sample Normalization") + norm_strategy = st.selectbox( + "Select Normalization", + options=["None", "sum", "median", "pqn", "reference_feature", "quantile"], + index=0, + help="Perform column-wise corrections to account for variable sample loading concentrations.", + ) + + # Conditionally display target input field for reference feature matching + ref_feature_input = None + if norm_strategy == "reference_feature": + ref_feature_input = st.text_input( + "Reference Protein Name (ID)", + value="", + placeholder="e.g., P01234 or GAPDH", + help=f"Enter the exact unique identifier string matching a key inside the '{id_col}' column.", + ) + +with col3: + st.markdown("### πŸ“Š 3. Row Scaling") + scaling_strategy = st.selectbox( + "Select Scaling Mode", + options=["None", "mean_centering", "auto_scaling", "pareto_scaling", "range_scaling"], + index=0, + help="Adjust individual feature weights to make low and high abundance proteins comparable.", + ) + + +# --- SECTION 3: Normalization Pipe Sequential Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Apply Normalization Pipelines", type="primary"): + + # Validate reference feature selection if active before hitting polars execution layers + if norm_strategy == "reference_feature" and not ref_feature_input: + st.error( + "❌ Validation Error: Please provide a valid Reference Protein Name to use the 'reference_feature' strategy." + ) + st.stop() + + # Convert pandas memory buffer into optimization lazy dataframe tree graph + processing_lazy = pl.from_pandas(base_df).lazy() + + # Execute Chain 1: Transform Matrix Data + try: + processing_lazy = transform_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=transform_strategy, + ) + + # Execute Chain 2: Normalize Sample Intensities (Columns) + processing_lazy = normalize_samples( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=norm_strategy, + id_col=id_col, + reference_feature=ref_feature_input if norm_strategy == "reference_feature" else None, + ) + + # Execute Chain 3: Scale Individual Features (Rows) + processing_lazy = scale_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=scaling_strategy, + ) + + # Finalize and collect pipeline query graph optimizations + normalized_df = strip_stat_columns(processing_lazy.collect().to_pandas()) + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (Statistics Block) + st.session_state["normalized_df"] = normalized_df + + st.success("Successfully executed all selected normalization pipelines!") + + # Display the finalized transformation matrix view + st.subheader("Normalized Abundance Table") + st.dataframe(normalized_df, use_container_width=True) + + except ValueError as val_err: + # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein) + st.error(f"Engine Configuration Error: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/content/results_abundance.py b/content/results_abundance.py index a7ff453..38c42bc 100644 --- a/content/results_abundance.py +++ b/content/results_abundance.py @@ -1,9 +1,11 @@ """Abundance (ProteomicsLFQ) Results Page.""" import streamlit as st import pandas as pd +import numpy as np from pathlib import Path from src.common.common import page_setup from src.common.results_helpers import get_workflow_dir, get_abundance_data +from src.workflow.ParameterManager import ParameterManager params = page_setup() st.title("Abundance Quantification") @@ -11,7 +13,7 @@ st.markdown( """ View protein and PSM-level quantification from **ProteomicsLFQ**. -This page calculates differential expression statistics between sample groups. +This page focuses on raw abundance intensity for preprocessing. """ ) @@ -21,6 +23,10 @@ workflow_dir = get_workflow_dir(st.session_state["workspace"]) quant_dir = workflow_dir / "results" / "quant_results" +parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") + +workflow_params = parameter_manager.get_parameters_from_json() +analysis_mode = workflow_params.get("analysis-mode", "LFQ") if not quant_dir.exists(): st.info("No quantification results available yet. Please run the workflow first.") @@ -35,6 +41,55 @@ csv_file = csv_files[0] +def render_protein_table(pivot_df, is_lfq=True): + """Common function to render the protein-level abundance table""" + pivot_df = pivot_df.copy() + st.markdown("### Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples." + ) + + if is_lfq: + # Handle LFQ mode columns (Raw Intensity) + id_col = "ProteinName" + exclude_cols = [id_col, "PeptideSequence"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + display_cols = [id_col, "Intensity"] + sample_cols + ["PeptideSequence"] + help_text = "Raw sample intensities" + y_min = None + else: + # Handle non-LFQ mode columns (Log2-transformed Intensity) + id_col = "protein" + exclude_cols = [id_col, "n_proteins", "n_peptides", "protein_score"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols and "ratio" not in c.lower()] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply( + lambda row: [np.log2(v + 1) for v in row], axis=1 + ) + display_cols = [id_col, "Intensity"] + sample_cols + help_text = "Sample intensities (log2 scale)" + y_min = 0 + + # Filter to available columns, then sort and display + available_cols = [c for c in display_cols if c in pivot_df.columns] + view_df = pivot_df[available_cols] + + st.dataframe( + view_df, + column_config={ + "Intensity": st.column_config.BarChartColumn( + "Intensity", + help=help_text, + width="small", + y_min=y_min, + ), + }, + use_container_width=True, + ) + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) try: @@ -44,68 +99,53 @@ st.info("No data found in this file.") st.stop() - with protein_tab: - st.markdown("### Protein-Level Abundance Table") + result = get_abundance_data(st.session_state["workspace"]) - st.info( - "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." - ) + if analysis_mode == "LFQ": + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) - result = get_abundance_data(st.session_state["workspace"]) - if result is None: - st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.") - st.page_link("content/workflow_configure.py", label="Go to Configure", icon="βš™οΈ") - st.stop() + with protein_tab: + if result is None: + st.warning("Could not load abundance data. Please run the workflow first.") + st.stop() + + pivot_df, expr_df, group_map = result + render_protein_table(pivot_df, is_lfq=True) - pivot_df, expr_df, group_map = result + with psm_tab: + st.markdown("### PSM-level Quantification Table") + st.info( + "This table shows the PSM-level quantification data, including protein IDs, " + "peptide sequences, charge states, and intensities across samples. " + "Each row represents one peptide-spectrum match detected from the MS/MS analysis." + ) + st.dataframe(df, use_container_width=True) - # Display group comparison info - groups = sorted(set(group_map.values())) - if len(groups) >= 2: - group1, group2 = sorted(groups)[:2] - st.info(f"Statistical comparison: **{group2} vs {group1}**") + else: + pre_processing_tab, protein_tab = st.tabs(["Pre-processing", "Protein Table"]) - # Get sample columns (between stats and PeptideSequence) - sample_cols = [c for c in pivot_df.columns if c not in ["ProteinName", "log2FC", "p-value", "PeptideSequence"]] + if result is None: + st.info("πŸ’‘ Please run the workflow first to see results.") + st.stop() - pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + pivot_df, expr_df, group_map = result - # Reorder columns: place Intensity after p-value - display_cols = ["ProteinName", "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"] - display_df = pivot_df[display_cols] - - st.dataframe( - display_df.sort_values("p-value"), - column_config={ - "Intensity": st.column_config.BarChartColumn( - "Intensity", - help="Raw sample intensities", - width="small", - ), - }, - use_container_width=True, - ) + with pre_processing_tab: + st.write("### Final Results (Intensity matrix)") + st.dataframe(pivot_df.head(10)) - with psm_tab: - st.markdown("### PSM-level Quantification Table") - st.info( - "This table shows the PSM-level quantification data, including protein IDs, " - "peptide sequences, charge states, and intensities across samples. " - "Each row represents one peptide-spectrum match detected from the MS/MS analysis." - ) - st.dataframe(df, use_container_width=True) + with protein_tab: + render_protein_table(pivot_df, is_lfq=False) except Exception as e: st.error(f"Failed to load {csv_file.name}: {e}") st.markdown("---") -st.markdown("**Next steps:** Explore statistical visualizations") +st.markdown("**Next steps:** Continue preprocessing, then run statistical inference") col1, col2, col3 = st.columns(3) with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") + st.page_link("content/filtering.py", label="Filtering", icon="🧹") with col2: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") + st.page_link("content/imputation.py", label="Imputation", icon="🧩") with col3: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") + st.page_link("content/statistical.py", label="Statistical Inference", icon="πŸ”¬") \ No newline at end of file diff --git a/content/results_heatmap.py b/content/results_heatmap.py index 4ece3f4..104bff6 100644 --- a/content/results_heatmap.py +++ b/content/results_heatmap.py @@ -1,19 +1,18 @@ """Heatmap Results Page.""" import streamlit as st import numpy as np -import plotly.express as px -from scipy.cluster.hierarchy import linkage, leaves_list -from scipy.spatial.distance import pdist +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import Heatmap params = page_setup() st.title("Heatmap") st.markdown( """ -Hierarchically clustered heatmap of protein-level abundance (Z-score normalized). -Proteins and samples are ordered by similarity. +Interactive hierarchically clustered heatmap of protein-level abundance (Z-score normalized). +Powered by OpenMS-Insight multi-resolution engine. """ ) @@ -28,42 +27,68 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) -top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +sample_cols = expr_df.columns.tolist() +# UI settings (number of top variance proteins) +top_n = st.slider("Number of proteins (Highest Variance)", 20, 200, 50, key="heatmap_top_n") + +# Process data (variance selection -> Z-score normalization) var_series = expr_df.var(axis=1) top_proteins = var_series.sort_values(ascending=False).head(top_n).index heatmap_df = expr_df.loc[top_proteins] + +# Compute Z-scores and clean missing/invalid values heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) + # Melt and convert data to Polars to satisfy OpenMS-Insight component requirements + # Restore the id column from the index as a regular column + heatmap_z_reset = heatmap_z.reset_index() - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) + # Unpivot the wide-format matrix into long-format (X, Y, Intensity) + melted_df = heatmap_z_reset.melt( + id_vars=[id_col], + value_vars=sample_cols, + var_name="Sample", + value_name="Z_score" + ) - heatmap_clustered = heatmap_z.iloc[row_order, col_order] + # Add sample group mapping if available for heatmap categories + if sample_group_map: + melted_df["Group"] = melted_df["Sample"].map(sample_group_map) - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) + # Pack the Pandas DataFrame into a Polars LazyFrame + heatmap_pl_lazy = pl.from_pandas(melted_df).lazy() - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} + # Initialize the OpenMS-Insight Heatmap component and map attributes + heatmap_component = Heatmap( + cache_id="quantms_protein_heatmap", + x_column="Sample", + y_column=id_col, + data=heatmap_pl_lazy, + intensity_column="Z_score", + title="Protein Abundance Heatmap (Z-score)", + x_label="Samples", + y_label="Proteins", + colorscale="RdBu", + reversescale=True, + log_scale=False, # Z-score can be negative, so log scale must stay off + intensity_label="Z-score", + category_column=None, + min_points=10000, # Generous point-count ceiling so the full grid renders ) - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, use_container_width=True) + # Render the component + state_manager = st.session_state.get("state") + heatmap_component(state_manager=state_manager) else: st.warning("Insufficient data to generate the heatmap.") diff --git a/content/results_heatmap_clustered.py b/content/results_heatmap_clustered.py new file mode 100644 index 0000000..7104c3a --- /dev/null +++ b/content/results_heatmap_clustered.py @@ -0,0 +1,107 @@ +"""Clustered Heatmap Results Page.""" +import streamlit as st +import numpy as np +import polars as pl +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import ClusteredHeatmap + +params = page_setup() +st.title("Clustered Heatmap") + +st.markdown( + """ +A real grid heatmap (rows = proteins, columns = samples) with hierarchical +clustering dendrograms on both axes and a sample-group color bar, powered +by OpenMS-Insight. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +top_n = st.slider("Number of proteins (Highest Variance)", 10, 200, 30, key="clustered_heatmap_top_n") + +var_series = expr_df.var(axis=1) +top_proteins = var_series.sort_values(ascending=False).head(top_n).index +heatmap_df = expr_df.loc[top_proteins] + +heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) +heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() + +if heatmap_z.empty: + st.warning("Insufficient data to generate the heatmap.") + st.stop() + +heatmap_z_reset = heatmap_z.reset_index() +heatmap_lazy = pl.from_pandas(heatmap_z_reset).lazy() + +sample_cols = expr_df.columns.tolist() +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, +) + +# Assign group annotation-bar colors in sorted-group order (matching how +# ClusteredHeatmap._preprocess() orders unique groups internally). +group_palette = [ + "#00BFC4", # teal + "#F8766D", # salmon + "#7CAE00", # yellow-green + "#C77CFF", # lavender purple + "#E7B800", # gold/amber + "#619CFF", # blue + "#FF61C3", # pink/magenta + "#00BA38", # green + "#FF8C42", # orange + "#00B0F6", # sky blue +] +unique_groups = sorted(set(sample_group_map.values())) +group_colors = {g: group_palette[i % len(group_palette)] for i, g in enumerate(unique_groups)} + +heatmap_component = ClusteredHeatmap( + cache_id="quantms_clustered_heatmap", + cache_path=str(st.session_state["workspace"]), + id_col=id_col, + data=heatmap_lazy, + metadata=metadata_pl, + row_cluster=True, + col_cluster=True, + title="Protein Abundance Heatmap (Z-score, clustered)", + x_label="Samples", + y_label="Proteins", + colorscale=[[0, "#6699E0"], [0.5, "#FFFFFF"], [1, "#E06666"]], + reversescale=False, + intensity_label="Z-score", + group_colors=group_colors, +) + +state_manager = st.session_state.get("state") +# Scale height with the number of proteins so row labels stay readable - +# BaseComponent otherwise defaults to a flat 400px, too short for a +# dendrogram+heatmap composite with more than a handful of rows. +heatmap_height = max(600, min(1400, 300 + top_n * 20)) +heatmap_component(state_manager=state_manager, height=heatmap_height) + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") +with col2: + st.page_link("content/results_heatmap.py", label="Heatmap (original)", icon="πŸ”₯") diff --git a/content/results_pathway_analysis.py b/content/results_pathway_analysis.py new file mode 100644 index 0000000..f5eb4c1 --- /dev/null +++ b/content/results_pathway_analysis.py @@ -0,0 +1,258 @@ +import json +import mygene +import streamlit as st +import pandas as pd +import numpy as np +import plotly.express as px +import plotly.io as pio +from collections import defaultdict +from scipy.stats import fisher_exact +from pathlib import Path +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data + +# ================================ +# Page setup +# ================================ +params = page_setup() +st.title("ProteomicsLFQ Results") + +# ================================ +# Workspace check +# ================================ +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# ================================ +# _run_go_enrichment function +# ================================ +def _run_go_enrichment(pivot_df: pd.DataFrame, results_dir: Path): + p_cutoff = 0.05 + fc_cutoff = 1.0 + + analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() + + if analysis_df.empty: + st.error("No valid statistical data found for GO enrichment.") + st.write("❗ analysis_df is empty") + else: + with st.spinner("Fetching GO terms from MyGene.info API..."): + mg = mygene.MyGeneInfo() + + def get_clean_uniprot(name): + parts = str(name).split("|") + return parts[1] if len(parts) >= 2 else parts[0] + + analysis_df["UniProt"] = analysis_df["protein"].apply(get_clean_uniprot) + + bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() + fg_ids = analysis_df[ + (analysis_df["p-value"] < p_cutoff) & + (analysis_df["log2FC"].abs() >= fc_cutoff) + ]["UniProt"].dropna().astype(str).unique().tolist() + # st.write("βœ… get_clean_uniprot applied") + + if len(fg_ids) < 3: + st.warning( + f"Not enough significant proteins " + f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " + f"Found: {len(fg_ids)}" + ) + st.write("❗ Not enough significant proteins") + else: + res_list = mg.querymany( + bg_ids, scopes="uniprot", fields="go", as_dataframe=False + ) + res_go = pd.DataFrame(res_list) + if "notfound" in res_go.columns: + res_go = res_go[res_go["notfound"] != True] + + def extract_go_terms(go_data, go_type): + if not isinstance(go_data, dict) or go_type not in go_data: + return [] + terms = go_data[go_type] + if isinstance(terms, dict): + terms = [terms] + return list({t.get("term") for t in terms if "term" in t}) + + for go_type in ["BP", "CC", "MF"]: + res_go[f"{go_type}_terms"] = res_go["go"].apply( + lambda x: extract_go_terms(x, go_type) + ) + + annotated_ids = set(res_go["query"].astype(str)) + fg_set = annotated_ids.intersection(fg_ids) + bg_set = annotated_ids + # st.write(f"βœ… fg_set bg_set are set") + + def run_go(go_type): + go2fg = defaultdict(set) + go2bg = defaultdict(set) + + for _, row in res_go.iterrows(): + uid = str(row["query"]) + for term in row[f"{go_type}_terms"]: + go2bg[term].add(uid) + if uid in fg_set: + go2fg[term].add(uid) + + records = [] + N_fg = len(fg_set) + N_bg = len(bg_set) + + for term, fg_genes in go2fg.items(): + a = len(fg_genes) + if a == 0: + continue + b = N_fg - a + c = len(go2bg[term]) - a + d = N_bg - (a + b + c) + + _, p = fisher_exact([[a, b], [c, d]], alternative="greater") + records.append({ + "GO_Term": term, + "Count": a, + "GeneRatio": f"{a}/{N_fg}", + "p_value": p, + }) + + df = pd.DataFrame(records) + if df.empty: + return None, None + + df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) + df = df.sort_values("p_value").head(20) + + # βœ… Plotly Figure + fig = px.bar( + df, + x="-log10(p)", + y="GO_Term", + orientation="h", + title=f"GO Enrichment ({go_type})", + ) + + # st.write(f"βœ… Plotly Figure generated") + + fig.update_layout( + yaxis=dict(autorange="reversed"), + height=500, + margin=dict(l=10, r=10, t=40, b=10), + ) + + return fig, df + + go_results = {} + + for go_type in ["BP", "CC", "MF"]: + fig, df_go = run_go(go_type) + if fig is not None: + go_results[go_type] = { + "fig": fig, + "df": df_go + } + # st.write(f"βœ… go_type generated") + + go_dir = results_dir / "go-terms" + go_dir.mkdir(parents=True, exist_ok=True) + + go_data = {} + + for go_type in ["BP", "CC", "MF"]: + if go_type in go_results: + fig = go_results[go_type]["fig"] + df = go_results[go_type]["df"] + + go_data[go_type] = { + "fig_json": fig.to_json(), # Figure β†’ JSON string + "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts + } + + go_json_file = go_dir / "go_results.json" + with open(go_json_file, "w") as f: + json.dump(go_data, f) + st.session_state["go_results"] = go_results + st.session_state["go_ready"] = True if go_data else False + # st.write("βœ… GO enrichment analysis complete") + +# ================================ +# Load abundance data +# ================================ +results_dir = Path(st.session_state["workspace"]) / "topp-workflow" / "results" / "quant_results" +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result + +go_json_file = results_dir / "go-terms" / "go_results.json" + +go_input_df = pivot_df.copy() +if "ProteinName" in go_input_df.columns: + go_input_df = go_input_df.rename(columns={"ProteinName": "protein"}) + +_run_go_enrichment(go_input_df, results_dir) + +# ================================ +# Tabs +# ================================ +protein_tab, = st.tabs(["🧬 Protein Table"]) + +# ================================ +# Protein-level results +# ================================ +with protein_tab: + st.markdown("### 🧬 Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples.\n\n" + "Additionally, log2 fold change and p-values are calculated between sample groups." + ) + + if pivot_df.empty: + st.info("No protein-level data available.") + else: + st.session_state["pivot_df"] = pivot_df + st.dataframe(pivot_df.sort_values("p-value"), width="stretch") + +# ====================================================== +# GO Enrichment Results +# ====================================================== +st.markdown("---") +st.subheader("🧬 GO Enrichment Analysis") + +if not go_json_file.exists(): + st.info("GO Enrichment results are not available yet. Please run the analysis first.") +else: + with open(go_json_file, "r") as f: + go_data = json.load(f) + + bp_tab, cc_tab, mf_tab = st.tabs([ + "🧬 Biological Process", + "🏠 Cellular Component", + "βš™οΈ Molecular Function", + ]) + + for tab, go_type in zip([bp_tab, cc_tab, mf_tab], ["BP", "CC", "MF"]): + with tab: + if go_type not in go_data: + st.info(f"No enriched {go_type} terms found.") + continue + + fig_json = go_data[go_type]["fig_json"] + df_dict = go_data[go_type]["df_dict"] + + fig = pio.from_json(fig_json) + + df_go = pd.DataFrame(df_dict) + + if df_go.empty: + st.info(f"No enriched {go_type} terms found.") + else: + st.plotly_chart(fig, width="stretch") + + st.markdown(f"#### {go_type} Enrichment Results") + st.dataframe(df_go, width="stretch") \ No newline at end of file diff --git a/content/results_pca.py b/content/results_pca.py index 45ea8eb..29f441a 100644 --- a/content/results_pca.py +++ b/content/results_pca.py @@ -1,11 +1,10 @@ """PCA Results Page.""" -import streamlit as st import pandas as pd -import plotly.express as px -from sklearn.decomposition import PCA -from sklearn.preprocessing import StandardScaler +import polars as pl +import streamlit as st from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import PCAPlot params = page_setup() st.title("PCA Analysis") @@ -13,7 +12,7 @@ st.markdown( """ Principal Component Analysis (PCA) of protein-level abundance. -Samples are colored by group assignment to visualize clustering. +Samples are projected onto their principal components and colored by group assignment to visualize clustering. """ ) @@ -21,6 +20,7 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Load abundance data (base wide-format table + sample -> group mapping) result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -28,60 +28,143 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +# Mirrors statistical.py: PCA should run on the most-processed data available. +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-adj", "stat", "p-value"] +] +unique_groups = sorted({sample_group_map[s] for s in sample_cols if s in sample_group_map}) + +if len(sample_cols) < 2: + st.info("PCA requires at least 2 samples.") + st.stop() -top_n = 500 +if len(unique_groups) < 2: + st.warning( + "Only one biological group was detected - points will still be plotted, " + "but group-based coloring requires 2 or more groups." + ) -top_proteins = ( - pivot_df - .dropna(subset=["p-adj"]) - .sort_values("p-adj", ascending=True) - .head(top_n)["ProteinName"] +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples " + f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})." ) +st.dataframe(base_df, use_container_width=True) -expr_df_pca = expr_df.loc[ - expr_df.index.intersection(top_proteins) -] +st.markdown("---") + +# --- SECTION 2: PCA Configuration --- +st.subheader("Configure PCA") + +expr_df_wide = base_df.set_index(id_col)[sample_cols] +max_available = expr_df_wide.shape[0] + +if max_available <= 20: + top_n = max_available + st.caption(f"Using all {top_n} proteins for PCA (dataset too small for variance filtering).") +else: + top_n = st.slider( + "Number of proteins (Highest Variance)", + min_value=20, + max_value=min(5000, max_available), + value=min(500, max_available), + step=10, + key="pca_top_n", + help=( + "PCA is computed only on the N proteins with the highest variance " + "across samples, to reduce noise from low-variance/uninformative features." + ), + ) + +top_proteins = expr_df_wide.var(axis=1).sort_values(ascending=False).head(top_n).index +expr_df_pca = expr_df_wide.loc[top_proteins].reset_index() if expr_df_pca.shape[0] < 2: - st.info("Not enough proteins after p-value filtering for PCA.") + st.info("Not enough proteins after variance filtering for PCA.") st.stop() -X = expr_df_pca.T -X_scaled = StandardScaler().fit_transform(X) - -pca = PCA(n_components=2) -pcs = pca.fit_transform(X_scaled) - -pca_df = pd.DataFrame( - pcs, - columns=["PC1", "PC2"], - index=X.index +# Prepare structural Polars metadata DataFrame required by PCAPlot +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, ) +pca_lazy = pl.from_pandas(expr_df_pca).lazy() + +# 3. Initialize the OpenMS-Insight PCAPlot component (computes PCA internally) +try: + pca_component = PCAPlot( + cache_id="quantms_pca_plot", + data=pca_lazy, + metadata=metadata_pl, + sample_id_field="sample_id", + group_field="group", + n_components=5, + title="Sample PCA", + ) +except ValueError as e: + st.error(f"PCA computation failed: {e}") + st.stop() -norm_map = { - k.replace(".mzML", ""): v - for k, v in group_map.items() -} -pca_df["Group"] = pca_df.index.map(norm_map) - -fig_pca = px.scatter( - pca_df, - x="PC1", - y="PC2", - color="Group", - text=pca_df.index, -) +variance_ratio = pca_component.get_variance_ratio() +pc_columns = pca_component.get_pc_columns() -fig_pca.update_traces(textposition="top center") -fig_pca.update_layout( - xaxis_title=f"PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)", - yaxis_title=f"PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)", - height=600, -) +# 4. Let the user pick which component pair to view (no recomputation needed) +col1, col2 = st.columns(2) +with col1: + pc_x_label = st.selectbox("X-axis component", pc_columns, index=0, key="pca_pc_x") +with col2: + default_y_index = 1 if len(pc_columns) > 1 else 0 + pc_y_label = st.selectbox("Y-axis component", pc_columns, index=default_y_index, key="pca_pc_y") + +pc_x = int(pc_x_label.replace("PC", "")) +pc_y = int(pc_y_label.replace("PC", "")) -st.plotly_chart(fig_pca, use_container_width=True) +# 5. Render the component +state_manager = st.session_state.get("state") +pca_component(state_manager=state_manager, pc_x=pc_x, pc_y=pc_y, height=600) -st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") +st.markdown( + "**Explained variance:** " + + ", ".join(f"{col} {ratio * 100:.1f}%" for col, ratio in zip(pc_columns, variance_ratio)) +) +st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by variance)") st.markdown("---") st.markdown("**Other visualizations:**") diff --git a/content/results_proteomicslfq.py b/content/results_proteomicslfq.py index 77eb332..fde2ab9 100644 --- a/content/results_proteomicslfq.py +++ b/content/results_proteomicslfq.py @@ -45,15 +45,14 @@ st.markdown("### 🧬 Protein-Level Abundance Table") st.info( "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." + "same protein and aggregating their intensities across samples." ) if pivot_df.empty: st.info("No protein-level data available.") else: st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df.sort_values("p-value"), use_container_width=True) + st.dataframe(pivot_df, use_container_width=True) # ====================================================== # GO Enrichment Results diff --git a/content/results_volcano.py b/content/results_volcano.py index 8502489..db2702f 100644 --- a/content/results_volcano.py +++ b/content/results_volcano.py @@ -1,9 +1,9 @@ """Volcano Plot Results Page.""" import streamlit as st -import plotly.express as px -import numpy as np +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column +from openms_insight import VolcanoPlot params = page_setup() st.title("Volcano Plot") @@ -19,6 +19,19 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Check if statistical analysis results are available in the session state +if "statistics_df" not in st.session_state or st.session_state["statistics_df"] is None: + st.info("Statistical analysis data not found. Please run the statistical engine first.") + st.page_link("content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬") + st.stop() + +# Retrieve the completed statistical analysis DataFrame +statistics_df = st.session_state["statistics_df"] + +if statistics_df.empty: + st.info("No data available for volcano plot.") + st.stop() + result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -26,16 +39,13 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) -if pivot_df.empty: - st.info("No data available for volcano plot.") - st.stop() - -volcano_df = pivot_df.copy() -volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"]) - -volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) +# 2. Clean data and convert to Polars for component input +volcano_df = statistics_df.dropna(subset=["log2FC", "p-adj"]).copy() +volcano_pl_lazy = pl.from_pandas(volcano_df).lazy() +# 3. Configure UI sliders (changing thresholds does not invalidate cache) fc_thresh = st.slider( "log2 Fold Change threshold", min_value=0.5, @@ -52,49 +62,34 @@ step=0.001, ) -volcano_df["Significance"] = "Not significant" -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", -] = "Up-regulated" - -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", -] = "Down-regulated" - -fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["ProteinName", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } +# 4. Initialize the OpenMS-Insight VolcanoPlot component +volcano_plot_component = VolcanoPlot( + cache_id="quantms_volcano_plot", + data=volcano_pl_lazy, + log2fc_column="log2FC", + pvalue_column="p-adj", + label_column=id_col, + up_color="#E74C3C", + down_color="#3498DB", + ns_color="#95A5A6", + show_threshold_lines=True, + threshold_line_style="dash", ) -fig_volcano.add_vline(x=fc_thresh, line_dash="dash") -fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") -fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - -# Make x-axis symmetric around zero -max_abs_fc = volcano_df["log2FC"].abs().max() -x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding +# 5. Render the component +state_manager = st.session_state.get("state") # Inject the project state management object -fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, +volcano_plot_component( + state_manager=state_manager, + fc_threshold=fc_thresh, + p_threshold=p_thresh, + max_labels=10, # Display labels for the top N significant proteins height=600, ) -st.plotly_chart(fig_volcano, use_container_width=True) - -up_count = (volcano_df["Significance"] == "Up-regulated").sum() -down_count = (volcano_df["Significance"] == "Down-regulated").sum() +# 6. Keep the existing statistical summary and bottom links +up_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh)).sum() +down_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh)).sum() st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") st.markdown("---") diff --git a/content/statistical.py b/content/statistical.py new file mode 100644 index 0000000..2e2a46d --- /dev/null +++ b/content/statistical.py @@ -0,0 +1,165 @@ +"""Statistical Inference Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import statistics engine functions from openms_insight +from openms_insight.analysis.statistics import calculate_statistical_tests, adjust_fdr_lazy + +params = page_setup() +st.title("Statistical Inference") + +st.markdown( + """ +Run differential expression analysis to identify statistically significant proteins across your biological groups. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract actual active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] +unique_groups = sorted(list(set([sample_group_map[s] for s in sample_cols if s in sample_group_map]))) +group_count = len(unique_groups) + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Dynamic Statistical Parameter Configuration --- +st.subheader("Configure Statistical Engine") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2 = st.columns(2) + +with col1: + st.markdown("### πŸ”¬ 1. Hypothesis Testing Method") + + # Route available method options dynamically based on the group count + if group_count == 2: + method_options = ["limma_like", "welch", "paired"] + help_text = "'limma_like' uses Empirical Bayes variance shrinking. 'welch' is for unequal variances. 'paired' is for dependent samples." + elif group_count >= 3: + method_options = ["limma_like", "anova"] + help_text = "'limma_like' supports multi-group design matrices. 'anova' computes standard row-wise One-way ANOVA." + else: + st.error("❌ Statistical testing requires at least 2 unique sample groups.") + st.stop() + + selected_method = st.selectbox( + "Select Statistical Test", + options=method_options, + index=0, + help=help_text + ) + +with col2: + st.markdown("### πŸ›‘οΈ 2. Multiple Testing Correction (FDR)") + selected_fdr = st.selectbox( + "Select FDR Adjustment Strategy", + options=["BH", "Bonferroni", "None"], + index=0, + help="'BH' (Benjamini-Hochberg) controls False Discovery Rate. 'Bonferroni' is strict Family-Wise Error Rate control." + ) + +# --- SECTION 3: Statistical Query Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Run Statistical Analysis", type="primary"): + + # Convert active pandas dataframe into polars lazyframe graph + stats_lazy = pl.from_pandas(base_df).lazy() + + try: + # Execute Chain 1: Calculate core statistics (Adds log2FC, stat, p-value) + stats_lazy = calculate_statistical_tests( + quantification_data=stats_lazy, + metadata=metadata_pl, + method=selected_method + ) + + # Execute Chain 2: Adjust Multiple Testing (Adds p-adj) + stats_lazy = adjust_fdr_lazy( + quantification_data=stats_lazy, + strategy=selected_fdr + ) + + # Resolve lazy graph optimization tree and bring back to pandas memory + statistics_df = stats_lazy.collect().to_pandas() + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (e.g., Volcano plot, Volcano/Heatmap UI) + st.session_state["statistics_df"] = statistics_df + + st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!") + + # Display the finalized statistics table view + st.subheader("Statistical Analysis Results") + st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`") + st.dataframe(statistics_df, use_container_width=True) + + except ValueError as val_err: + st.error(f"Engine Validation Fallure: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index 2af9bda..120efb0 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -1,9 +1,10 @@ import streamlit as st from pathlib import Path +import re import pandas as pd import plotly.express as px -#from streamlit_plotly_events import plotly_events -#from pyopenms import IdXMLFile +from streamlit_plotly_events import plotly_events +from pyopenms import IdXMLFile from scipy.stats import ttest_ind import numpy as np import mygene @@ -14,7 +15,7 @@ from src.common.common import page_setup from src.common.results_helpers import get_abundance_data from src.common.results_helpers import parse_idxml, build_spectra_cache -#from openms_insight import Table, Heatmap, LinePlot, SequenceView +from openms_insight import Table, Heatmap, LinePlot, SequenceView # params = page_setup() class WorkflowTest(WorkflowManager): @@ -47,6 +48,29 @@ def configure(self) -> None: self.ui.select_input_file("mzML-files", multiple=True, reactive=True) self.ui.select_input_file("fasta-file", multiple=False) + self.params = self.parameter_manager.get_parameters_from_json() + saved_mode = self.params.get("analysis-mode", "LFQ") + + self.ui.input_widget( + key="analysis-mode", + default=saved_mode, + name="Analysis Mode", + widget_type="selectbox", + options=["LFQ", "TMT"], + help="Choose between Label-Free Quantification (LFQ) or Tandem Mass Tag (TMT) analysis.", + reactive=True + ) + + self.params = self.parameter_manager.get_parameters_from_json() + current_mode = self.params.get("analysis-mode", "LFQ") + + if current_mode == "LFQ": + self.render_lfq_tabs() + else: + self.render_tmt_tabs() + + def render_lfq_tabs(self): + st.subheader("LFQ Analysis Mode") t = st.tabs(["**Identification**", "**Rescoring**", "**Filtering**", "**Library Generation**", "**Quantification**", "**Group Selection**"]) with t[0]: @@ -70,10 +94,10 @@ def configure(self) -> None: st.info(""" **Decoy Database Settings:** * **method**: How decoy sequences are generated from target protein sequences. - *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly - rearranges the amino acids. Both methods preserve the amino acid composition - of the original protein, ensuring decoys have similar properties to real sequences - for accurate false discovery rate (FDR) estimation. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. """) self.ui.input_TOPP( "DecoyDatabase", @@ -102,7 +126,7 @@ def configure(self) -> None: st.info(comet_info) comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] if not self.params.get("generate-decoys", True): # Only show decoy_string when not generating decoys comet_include.append("PeptideIndexing:decoy_string") @@ -111,7 +135,7 @@ def configure(self) -> None: "CometAdapter", custom_defaults={ "threads": 8, - "instrument": "high_res", + "instrument": "low_res", "missed_cleavages": 2, "min_peptide_length": 6, "max_peptide_length": 40, @@ -120,17 +144,19 @@ def configure(self) -> None: "isotope_error": "0/1", "precursor_charge": "2:4", "precursor_mass_tolerance": 20.0, - "fragment_mass_tolerance": 0.02, - "fragment_bin_offset": 0.0, + "fragment_mass_tolerance": 0.6, + "fragment_bin_offset": 0.4, "max_variable_mods_in_peptide": 3, "minimum_peaks": 1, "clip_nterm_methionine": "true", - "PeptideIndexing:IL_equivalent": "true", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)", + "PeptideIndexing:IL_equivalent": True, "PeptideIndexing:unmatched_action": "warn", "PeptideIndexing:decoy_string": "rev_", + "mass_recalibration": False, }, - flag_parameters=["PeptideIndexing:IL_equivalent"], include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "mass_recalibration"], exclude_parameters=["second_enzyme"], ) @@ -151,10 +177,10 @@ def configure(self) -> None: "subset_max_train": 300000, "decoy_pattern": "rev_", "score_type": "pep", - "post_processing_tdc": "true", + "post_processing_tdc": True, }, - flag_parameters=["post_processing_tdc"], include_parameters=percolator_include, + flag_parameters=["post_processing_tdc"], exclude_parameters=["out_type"], ) @@ -250,6 +276,10 @@ def configure(self) -> None: "psmFDR": 0.01, "proteinFDR": 0.01, "picked_proteinFDR": "true", + "alignment_order": "star", + "protein_quantification": "unique_peptides", + "quantification_method": "feature_intensity", + "protein_inference": "aggregation", }, include_parameters=["intThreshold", "psmFDR", "proteinFDR"], ) @@ -300,6 +330,294 @@ def configure(self) -> None: if orphaned_keys: self.parameter_manager.save_parameters() + def render_tmt_tabs(self): + st.subheader("TMT Analysis Mode") + # Create tabs for different analysis steps. + t = st.tabs( + ["**IsobaricAnalyzer**", "**CometAdapter**", "**PercolatorAdapter**", "**IDFilter**", "**IDMapper**", "**FileMerger**", + "**ProteinInference**", "**IDFilter**", "**IDConflictResolver**", "**ProteinQuantifier**", "**Group Selection**"] + ) + with t[0]: + # Checkbox for decoy generation + # reactive=True ensures the parent configure() fragment re-runs when checkbox changes, + # so conditional UI (DecoyDatabase settings) updates immediately + self.ui.input_widget( + key="generate-decoys", + default=True, + name="Generate Decoy Database", + widget_type="checkbox", + help="Generate reversed decoy sequences for FDR calculation. Disable if your FASTA already contains decoys.", + reactive=True, + ) + + # Reload params to get current checkbox value after it was saved + self.params = self.parameter_manager.get_parameters_from_json() + + # Show DecoyDatabase settings if generating decoys + if self.params.get("generate-decoys", True): + st.info(""" + **Decoy Database Settings:** + * **method**: How decoy sequences are generated from target protein sequences. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. + """) + self.ui.input_TOPP( + "DecoyDatabase", + custom_defaults={ + "decoy_string": "rev_", + "decoy_string_position": "prefix", + "method": "reverse", + }, + include_parameters=["method"], + ) + + comet_info = """ + **Identification (Comet):** + * **enzyme**: The enzyme used for peptide digestion. + * **missed_cleavages**: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. + * **fixed_modifications**: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **variable_modifications**: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **instrument**: Type of instrument (high_res or low_res). Use 'high_res' for high-resolution MS2 (Orbitrap, TOF), 'low_res' for ion trap. + * **fragment_mass_tolerance**: Fragment mass tolerance for MS2 matching. + * **fragment_bin_offset**: Offset for binning MS2 spectra. Typically 0.0 for high-res, 0.4 for low-res instruments. + """ + if not self.params.get("generate-decoys", True): + comet_info += """* **PeptideIndexing:decoy_string**: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions + in the protein database to indicate decoy proteins. + """ + st.info(comet_info) + + st.write(Path(self.workflow_dir, "results")) + + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + if not self.params.get("generate-decoys", True): + # Only show decoy_string when not generating decoys + comet_include.append("PeptideIndexing:decoy_string") + + self.ui.input_TOPP( + "IsobaricAnalyzer", + custom_defaults={ + "tmt11plex:reference_channel": 126, + "type": "tmt11plex", + "extraction:select_activation": "auto", + "extraction:reporter_mass_shift": 0.002, + "extraction:min_reporter_intensity": 0.0, + "extraction:min_precursor_purity": 0.0, + "extraction:precursor_isotope_deviation": 10.0, + "quantification:isotope_correction": "false", + }, + tool_instance_name="IsobaricAnalyzer-TMT", + reactive=True, + ) + with t[1]: + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset", "PeptideIndexing:IL_equivalent"] + self.ui.input_TOPP( + "CometAdapter", + custom_defaults={ + "PeptideIndexing:IL_equivalent": True, + "clip_nterm_methionine": "true", + "instrument": "high_res", + "missed_cleavages": 2, + "min_peptide_length": 6, + "max_peptide_length": 40, + "enzyme": "Trypsin/P", + "PeptideIndexing:unmatched_action": "warn", + "max_variable_mods_in_peptide": 3, + "precursor_mass_tolerance": 4.5, + "isotope_error": "0/1", + "precursor_error_units": "ppm", + "num_hits": 1, + "num_enzyme_termini": "fully", + "fragment_bin_offset": 0.0, + "minimum_peaks": 10, + "precursor_charge": "2:4", + "fragment_mass_tolerance": 0.015, + "PeptideIndexing:unmatched_action": "warn", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)\nTMT6plex (K)\nTMT6plex (N-term)", + "debug": 0, + "force": True, + }, + include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "force"], + exclude_parameters=["second_enzyme"], + tool_instance_name="CometAdapter-TMT", + ) + with t[2]: + st.info(""" + **Filtering (IDFilter):** + * **score:type_peptide**: Score used for filtering. If empty, the main score is used. + * **score:psm**: The score which should be reached by a peptide hit to be kept. (use 'NAN' to disable this filter) + """) + self.ui.input_TOPP( + "PercolatorAdapter", + custom_defaults={ + "subset_max_train": 300000, + "decoy_pattern": "DECOY_", + "score_type": "pep", + "post_processing_tdc": True, + "debug": 0, + }, + flag_parameters=["post_processing_tdc"], + tool_instance_name="PercolatorAdapter-TMT", + ) + + with t[3]: + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_peptide": "q-value", + "score:psm": 0.10, + }, + tool_instance_name="IDFilter-strict", + ) + with t[4]: + st.info(""" + **Quantification (ProteomicsLFQ):** + * **intThreshold**: Peak intensity threshold applied in seed detection. + * **psmFDR**: FDR threshold for sub-protein level (e.g. 0.05=5%). Use -FDR_type to choose the level. Cutoff is applied at the highest level. If Bayesian inference was chosen, it is equivalent with a peptide FDR + * **proteinFDR**: Protein FDR threshold (0.05=5%). + """) + self.ui.input_TOPP( + "IDMapper", + custom_defaults={ + "threads": 8, + "debug": 0, + }, + tool_instance_name="IDMapper-TMT", + ) + with t[5]: + self.ui.input_TOPP( + "FileMerger", + custom_defaults={ + "in_type": "consensusXML", + "append_method": "append_cols", + "annotate_file_origin": True, + "threads": 8, + }, + flag_parameters=["annotate_file_origin"], + tool_instance_name="FileMerger-TMT", + ) + with t[6]: + self.ui.input_TOPP( + "ProteinInference", + custom_defaults={ + "threads": 8, + "picked_decoy_string": "DECOY_", + "picked_fdr": "true", + "protein_fdr": "true", + "Algorithm:use_shared_peptides": "true", + "Algorithm:annotate_indistinguishable_groups": "true", + "Algorithm:score_type": "PEP", + "Algorithm:score_aggregation_method": "best", + "Algorithm:min_peptides_per_protein": 1, + }, + tool_instance_name="ProteinInference-TMT", + ) + with t[7]: + # A single checkbox widget for workflow logic. + # self.ui.input_widget("run-python-script", False, "Run custom Python script") * + # Generate input widgets for a custom Python tool, located at src/python-tools. + # Parameters are specified within the file in the DEFAULTS dictionary. + # self.ui.input_python("example") * + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_protein": "q-value", + "score:proteingroup": 0.01, + "score:psm": 0.01, + "delete_unreferenced_peptide_hits": True, + "remove_decoys": True + }, + flag_parameters=["delete_unreferenced_peptide_hits", "remove_decoys"], + tool_instance_name="IDFilter-lenient", + ) + with t[8]: + self.ui.input_TOPP( + "IDConflictResolver", + custom_defaults={ + "threads": 4, + }, + tool_instance_name="IDConflictResolver-TMT", + ) + + with t[9]: + self.ui.input_TOPP( + "ProteinQuantifier", + custom_defaults={ + "method": "top", + "top:N": 3, + "top:aggregate": "median", + "top:include_all": True, + "ratios": True, + "threads": 8, + "debug": 0, + }, + flag_parameters=["top:include_all", "ratios"], + tool_instance_name="ProteinQuantifier-TMT", + ) + with t[10]: + st.markdown("### πŸ§ͺ TMT Sample Group Assignment") + + latest_params = self.parameter_manager.get_parameters_from_json() + type_key = ( + f"{self.parameter_manager.topp_param_prefix}" + "IsobaricAnalyzer-TMT:1:type" + ) + selected_type = str( + st.session_state.get(type_key) + or latest_params.get("IsobaricAnalyzer-TMT", {}).get("type") + or "tmt11plex" + ).lower() + + m = re.search(r'\d+', selected_type) + is_supported_type = any(label in selected_type for label in ["tmt", "itraq"]) + if not m or not is_supported_type: + st.warning("Please select a supported isobaric type in the IsobaricAnalyzer tab first.") + else: + num_plex = int(m.group()) + channels = [f"sample{i+1}" for i in range(num_plex)] + st.caption(f"Isobaric type: **{selected_type}** - {num_plex} channels") + st.info("Assign a group name to each channel. Use **'skip'** to exclude a channel.") + + for row_start in range(0, num_plex, 2): + c1, c2 = st.columns(2) + + left_idx = row_start + left_channel = channels[left_idx] + with c1: + self.ui.input_widget( + key=f"TMT-group-{left_channel}", + default="", + name=f"Group for channel {left_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + right_idx = row_start + 1 + if right_idx < num_plex: + right_channel = channels[right_idx] + with c2: + self.ui.input_widget( + key=f"TMT-group-{right_channel}", + default="", + name=f"Group for channel {right_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + # Remove orphaned params from a previously selected larger plex + self.params = self.parameter_manager.get_parameters_from_json() + valid_keys = {f"TMT-group-{ch}" for ch in channels} + orphaned = [k for k in self.params if k.startswith("TMT-group-") and k not in valid_keys] + if orphaned: + for k in orphaned: + del self.params[k] + self.parameter_manager.save_parameters() + def execution(self) -> bool: """ Refactored TOPP workflow execution: @@ -352,639 +670,944 @@ def execution(self) -> bool: st.info(f"Using original FASTA: {fasta_path.name}") database_fasta = fasta_path - # ================================ - # 1️⃣ Directory setup - # ================================ - results_dir = Path(self.workflow_dir, "results") - comet_dir = results_dir / "comet_results" - perc_dir = results_dir / "percolator_results" - filter_dir = results_dir / "filter_results" - quant_dir = results_dir / "quant_results" - - for d in [comet_dir, perc_dir, filter_dir, quant_dir]: - d.mkdir(parents=True, exist_ok=True) - - self.logger.log("πŸ“ Output directories created") - - # # ================================ - # # 2️⃣ File path definitions (per sample) - # # ================================ - comet_results = [] - percolator_results = [] - filter_results = [] - - for mz in in_mzML: - stem = Path(mz).stem - comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) - percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) - filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + current_mode = self.params.get("analysis-mode", "LFQ") + st.write(f"Current analysis mode: **{current_mode}**") - # ================================ - # 3️⃣ Per-file processing - # ================================ - for i, mz in enumerate(in_mzML): - stem = Path(mz).stem - st.info(f"Processing sample: {stem}") + if current_mode == "LFQ": + self.logger.log("βš™οΈ Running LFQ workflow") - self.logger.log("πŸ”¬ Starting per-sample processing...") + # ================================ + # 1️⃣ Directory setup + # ================================ + results_dir = Path(self.workflow_dir, "results") + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + filter_dir = results_dir / "psm_filter" + quant_dir = results_dir / "quant_results" + + results_dir = Path(self.workflow_dir, "input-files") + + for d in [comet_dir, perc_dir, filter_dir, quant_dir]: + d.mkdir(parents=True, exist_ok=True) + + self.logger.log("πŸ“ Output directories created") + + # ================================ + # 2️⃣ File path definitions (per sample) + # ================================ + comet_results = [] + percolator_results = [] + filter_results = [] + + for mz in in_mzML: + stem = Path(mz).stem + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) + filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + + # ================================ + # 3️⃣ Per-file processing + # ================================ + for i, mz in enumerate(in_mzML): + stem = Path(mz).stem + st.info(f"Processing sample: {stem}") + + self.logger.log("πŸ”¬ Starting per-sample processing...") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - # --- CometAdapter --- - self.logger.log("πŸ”Ž Running peptide search...") - with st.spinner(f"CometAdapter ({stem})"): - comet_extra_params = {"database": str(database_fasta)} - if self.params.get("generate-decoys", True): - # Propagate decoy_string from DecoyDatabase - comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + ): + self.logger.log("Workflow stopped due to error") + return False - if not self.executor.run_topp( - "CometAdapter", - { - "in": in_mzML, - "out": comet_results, - }, - comet_extra_params, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Get fragment tolerance from CometAdapter parameters for visualization - comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") - frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) - frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" - - # Build visualization cache for Comet results - results_dir_path = Path(self.workflow_dir, "results") - cache_dir = results_dir_path / "insight_cache" - cache_dir.mkdir(parents=True, exist_ok=True) - - # Get mzML directory - mzml_dir = Path(in_mzML[0]).parent - - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} - - for idxml_file in comet_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - self.logger.log("βœ… Peptide search complete") + self.logger.log("βœ… Peptide search complete") - # --- PercolatorAdapter --- - self.logger.log("πŸ“Š Running rescoring...") - with st.spinner(f"PercolatorAdapter ({stem})"): - if not self.executor.run_topp( - "PercolatorAdapter", - { - "in": comet_results, - "out": percolator_results, - }, - {"decoy_pattern": decoy_string}, # Always propagated from upstream - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Percolator results - for idxml_file in percolator_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # if not Path(comet_results).exists(): + # st.error(f"CometAdapter failed for {stem}") + # st.stop() - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter ({stem})"): + if not self.executor.run_topp( + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + {"decoy_pattern": decoy_string}, # Always propagated from upstream + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - self.logger.log("βœ… Rescoring complete") + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # if not Path(percolator_results[i]).exists(): - # st.error(f"PercolatorAdapter failed for {stem}") - # st.stop() + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - # --- IDFilter --- - self.logger.log("πŸ”§ Filtering identifications...") - with st.spinner(f"IDFilter ({stem})"): - if not self.executor.run_topp( - "IDFilter", - { - "in": percolator_results, - "out": filter_results, - }, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Filter results - for idxml_file in filter_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + self.logger.log("βœ… Rescoring complete") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # if not Path(percolator_results[i]).exists(): + # st.error(f"PercolatorAdapter failed for {stem}") + # st.stop() - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter ({stem})"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": filter_results, + }, + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Build visualization cache for Filter results + for idxml_file in filter_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - self.logger.log("βœ… Filtering complete") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # if not Path(filter_results[i]).exists(): - # st.error(f"IDFilter failed for {stem}") - # st.stop() + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # ================================ - # EasyPQP Spectral Library Generation (optional) - # ================================ - if self.params.get("generate-library", False): - self.logger.log("πŸ“š Building spectral library with EasyPQP...") - st.info("Building spectral library with EasyPQP...") - library_dir = Path(self.workflow_dir, "results", "library") - library_dir.mkdir(parents=True, exist_ok=True) - - psms_files, peaks_files = [], [] - - for filter_idxml in filter_results: - original_stem = Path(filter_idxml).stem.replace("_filter", "") - matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) - if not matching_mzml: - self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") - continue - - # easypqp library requires specific extensions for file recognition: - # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension - # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension - # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name - psms_out = str(library_dir / f"{original_stem}.psmpkl") - peaks_out = str(library_dir / f"{original_stem}.peakpkl") - - convert_cmd = [ - "easypqp", "convert", - "--pepxml", filter_idxml, - "--spectra", matching_mzml, - "--psms", psms_out, - "--peaks", peaks_out - ] - if self.executor.run_command(convert_cmd): - psms_files.append(psms_out) - peaks_files.append(peaks_out) - - if psms_files: - # easypqp library outputs TSV format (despite common .pqp extension) - library_tsv = str(library_dir / "spectral_library.tsv") - library_cmd = ["easypqp", "library", "--out", library_tsv] - - if not self.params.get("library-use-fdr", False): - # --nofdr only skips FDR recalculation, NOT threshold filtering - # Set all thresholds to 1.0 to bypass filtering for pre-filtered input - library_cmd.extend([ - "--nofdr", - "--psm_fdr_threshold", "1.0", - "--peptide_fdr_threshold", "1.0", - "--protein_fdr_threshold", "1.0" - ]) - else: - # Apply user-specified FDR filtering - library_cmd.extend([ - "--psm_fdr_threshold", - str(self.params.get("library-psm-fdr", 0.01)), - "--peptide_fdr_threshold", - str(self.params.get("library-peptide-fdr", 0.01)), - "--protein_fdr_threshold", - str(self.params.get("library-protein-fdr", 0.01)) - ]) - - for psms, peaks in zip(psms_files, peaks_files): - library_cmd.extend([psms, peaks]) - - if self.executor.run_command(library_cmd): - self.logger.log("βœ… Spectral library created") - st.success("Spectral library created") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + self.logger.log("βœ… Filtering complete") + + # if not Path(filter_results[i]).exists(): + # st.error(f"IDFilter failed for {stem}") + # st.stop() + + # ================================ + # EasyPQP Spectral Library Generation (optional) + # ================================ + if self.params.get("generate-library", False): + self.logger.log("πŸ“š Building spectral library with EasyPQP...") + st.info("Building spectral library with EasyPQP...") + library_dir = Path(self.workflow_dir, "results", "library") + library_dir.mkdir(parents=True, exist_ok=True) + + psms_files, peaks_files = [], [] + + for filter_idxml in filter_results: + original_stem = Path(filter_idxml).stem.replace("_filter", "") + matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) + if not matching_mzml: + self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") + continue + + # easypqp library requires specific extensions for file recognition: + # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension + # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension + # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name + psms_out = str(library_dir / f"{original_stem}.psmpkl") + peaks_out = str(library_dir / f"{original_stem}.peakpkl") + + convert_cmd = [ + "easypqp", "convert", + "--pepxml", filter_idxml, + "--spectra", matching_mzml, + "--psms", psms_out, + "--peaks", peaks_out + ] + if self.executor.run_command(convert_cmd): + psms_files.append(psms_out) + peaks_files.append(peaks_out) + + if psms_files: + # easypqp library outputs TSV format (despite common .pqp extension) + library_tsv = str(library_dir / "spectral_library.tsv") + library_cmd = ["easypqp", "library", "--out", library_tsv] + + if not self.params.get("library-use-fdr", False): + # --nofdr only skips FDR recalculation, NOT threshold filtering + # Set all thresholds to 1.0 to bypass filtering for pre-filtered input + library_cmd.extend([ + "--nofdr", + "--psm_fdr_threshold", "1.0", + "--peptide_fdr_threshold", "1.0", + "--protein_fdr_threshold", "1.0" + ]) + else: + # Apply user-specified FDR filtering + library_cmd.extend([ + "--psm_fdr_threshold", + str(self.params.get("library-psm-fdr", 0.01)), + "--peptide_fdr_threshold", + str(self.params.get("library-peptide-fdr", 0.01)), + "--protein_fdr_threshold", + str(self.params.get("library-protein-fdr", 0.01)) + ]) + + for psms, peaks in zip(psms_files, peaks_files): + library_cmd.extend([psms, peaks]) + + if self.executor.run_command(library_cmd): + self.logger.log("βœ… Spectral library created") + st.success("Spectral library created") + else: + self.logger.log("Warning: Failed to build spectral library") else: - self.logger.log("Warning: Failed to build spectral library") - else: - self.logger.log("Warning: No PSMs converted for library generation") + self.logger.log("Warning: No PSMs converted for library generation") + + st.success(f"βœ“ {stem} identification completed") + + # # ================================ + # # 4️⃣ ProteomicsLFQ (cross-sample) + # # ================================ + self.logger.log("πŸ“ˆ Running cross-sample quantification...") + st.info("Running ProteomicsLFQ (cross-sample quantification)") + + quant_mztab = str(quant_dir / "openms_quant.mzTab") + quant_cxml = str(quant_dir / "openms.consensusXML") + quant_msstats = str(quant_dir / "openms_msstats.csv") + + with st.spinner("ProteomicsLFQ"): + combined_in = " ".join(in_mzML) + combined_ids = " ".join(filter_results) + self.logger.log(f"COMBINED_IN {combined_in}", 1) + self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) + self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) + self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + + # βœ… Streamlit output (debug view) + st.markdown("### πŸ” ProteomicsLFQ Input Debug") + st.write("**combined_in:**", combined_in) + st.write("**combined_in type:**", type(combined_in).__name__) + + st.write("**combined_ids:**", combined_ids) + st.write("**combined_ids type:**", type(combined_ids).__name__) + + if not self.executor.run_topp( + "ProteomicsLFQ", + { + "in": [in_mzML], + "ids": [filter_results], + "out": [quant_mztab], + "out_cxml": [quant_cxml], + "out_msstats": [quant_msstats], + }, + { + "fasta": str(database_fasta), + "threads": 12, + # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 + "PeptideQuantification:extract:IM_window": "0.0", + "PeptideQuantification:faims:merge_features": "false", + }, + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… Quantification complete") + + # if not Path(quant_mztab).exists(): + # st.error("ProteomicsLFQ failed: mzTab not created") + # st.stop() + + # ================================ + # 5️⃣ Final report + # # ================================ + st.success("πŸŽ‰ TOPP workflow completed successfully") + st.write("πŸ“ Results directory:") + st.code(str(results_dir)) + + st.write("πŸ“„ Generated files:") + st.write(f"- mzTab: {quant_mztab}") + st.write(f"- consensusXML: {quant_cxml}") + st.write(f"- MSstats CSV: {quant_msstats}") - st.success(f"βœ“ {stem} identification completed") + return True + else: + self.logger.log("βš™οΈ Running TMT workflow") - # ================================ - # 4️⃣ ProteomicsLFQ (cross-sample) - # ================================ - self.logger.log("πŸ“ˆ Running cross-sample quantification...") - st.info("Running ProteomicsLFQ (cross-sample quantification)") + results_dir = Path(self.workflow_dir, "results") + iso_dir = results_dir / "isobaric_consensusXML" + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + psm_filter_dir = results_dir / "psm_filter" + map_dir = results_dir / "idmapper" + merge_dir = results_dir / "merged" + protein_dir = results_dir / "protein" + msstats_dir = results_dir / "msstats" + quant_dir = results_dir / "quant_results" + + iso_consensus = [] + comet_results = [] + percolator_results = [] + psm_filtered = [] + mapped_ids = [] + + for d in [ + iso_dir, comet_dir, perc_dir, psm_filter_dir, + map_dir, merge_dir, protein_dir, msstats_dir, quant_dir + ]: + d.mkdir(parents=True, exist_ok=True) + + for mz in in_mzML: + stem = Path(mz).stem + iso_consensus.append(str(iso_dir / f"{stem}_iso.consensusXML")) + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_comet_perc.idXML")) + psm_filtered.append(str(psm_filter_dir / f"{stem}_comet_perc_filter.idXML")) + mapped_ids.append(str(map_dir / f"{stem}_comet_perc_filter_map.consensusXML")) + + merged_id = str(merge_dir / "ID_mapper_merge.consensusXML") + protein_id = str(protein_dir / "ID_mapper_merge_epi.consensusXML") + protein_filter = str(protein_dir / "ID_mapper_merge_epi_filter.consensusXML") + protein_resolved = str(protein_dir / "ID_mapper_merge_epi_filter_resconf.consensusXML") + consensus_out = str(quant_dir / "openms_design_protein_openms.csv") + + # --- IsobaricAnalyzer --- + self.logger.log("🏷️ Running isobaric labeling analysis...") + with st.spinner("IsobaricAnalyzer"): + if not self.executor.run_topp( + "IsobaricAnalyzer", + { + "in": in_mzML, + "out": iso_consensus, + }, + tool_instance_name="IsobaricAnalyzer-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IsobaricAnalyzer complete") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + tool_instance_name="CometAdapter-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… CometAdapter complete") + + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - quant_mztab = str(quant_dir / "openms_quant.mzTab") - quant_cxml = str(quant_dir / "openms.consensusXML") - quant_msstats = str(quant_dir / "openms_msstats.csv") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - with st.spinner("ProteomicsLFQ"): - combined_in = " ".join(in_mzML) - combined_ids = " ".join(filter_results) - self.logger.log(f"COMBINED_IN {combined_in}", 1) - self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) - self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) - self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # βœ… Streamlit output (debug view) - st.markdown("### πŸ” ProteomicsLFQ Input Debug") - st.write("**combined_in:**", combined_in) - st.write("**combined_in type:**", type(combined_in).__name__) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - st.write("**combined_ids:**", combined_ids) - st.write("**combined_ids type:**", type(combined_ids).__name__) + self.logger.log("βœ… Peptide search complete") + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter"): if not self.executor.run_topp( - "ProteomicsLFQ", - { - "in": [in_mzML], - "ids": [filter_results], - "out": [quant_mztab], - "out_cxml": [quant_cxml], - "out_msstats": [quant_msstats], - }, - { - "fasta": str(database_fasta), - "psmFDR": 0.5, - "proteinFDR": 0.5, - "threads": 12, - # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 - "PeptideQuantification:extract:IM_window": "0.0", - "PeptideQuantification:faims:merge_features": "false", - } - ): + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + tool_instance_name="PercolatorAdapter-TMT", + ): self.logger.log("Workflow stopped due to error") return False - self.logger.log("βœ… Quantification complete") - - # ====================================================== - # ⚠️ 5️⃣ GO Enrichment Analysis (INLINE IN EXECUTION) - # ====================================================== - workspace_path = Path(self.workflow_dir).parent - res = get_abundance_data(workspace_path) - if res is not None: - pivot_df, _, _ = res - self.logger.log("βœ… pivot_df loaded, starting GO enrichment...") - self._run_go_enrichment(pivot_df, results_dir) - else: - st.warning("GO enrichment skipped: abundance data not available.") + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # ================================ - # 5️⃣ Final report - # # ================================ - st.success("πŸŽ‰ TOPP workflow completed successfully") - st.write("πŸ“ Results directory:") - st.code(str(results_dir)) - - return True - - def _run_go_enrichment(self, pivot_df: pd.DataFrame, results_dir: Path): - p_cutoff = 0.05 - fc_cutoff = 1.0 - - analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() - - if analysis_df.empty: - st.error("No valid statistical data found for GO enrichment.") - self.logger.log("❗ analysis_df is empty") - else: - with st.spinner("Fetching GO terms from MyGene.info API..."): - mg = mygene.MyGeneInfo() - - def get_clean_uniprot(name): - parts = str(name).split("|") - return parts[1] if len(parts) >= 2 else parts[0] - - analysis_df["UniProt"] = analysis_df["ProteinName"].apply(get_clean_uniprot) - - bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() - fg_ids = analysis_df[ - (analysis_df["p-value"] < p_cutoff) & - (analysis_df["log2FC"].abs() >= fc_cutoff) - ]["UniProt"].dropna().astype(str).unique().tolist() - self.logger.log("βœ… get_clean_uniprot applied") - - if len(fg_ids) < 3: - st.warning( - f"Not enough significant proteins " - f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " - f"Found: {len(fg_ids)}" - ) - self.logger.log("❗ Not enough significant proteins") - else: - res_list = mg.querymany( - bg_ids, scopes="uniprot", fields="go", as_dataframe=False - ) - res_go = pd.DataFrame(res_list) - if "notfound" in res_go.columns: - res_go = res_go[res_go["notfound"] != True] - - def extract_go_terms(go_data, go_type): - if not isinstance(go_data, dict) or go_type not in go_data: - return [] - terms = go_data[go_type] - if isinstance(terms, dict): - terms = [terms] - return list({t.get("term") for t in terms if "term" in t}) - - for go_type in ["BP", "CC", "MF"]: - res_go[f"{go_type}_terms"] = res_go["go"].apply( - lambda x: extract_go_terms(x, go_type) - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - annotated_ids = set(res_go["query"].astype(str)) - fg_set = annotated_ids.intersection(fg_ids) - bg_set = annotated_ids - self.logger.log(f"βœ… fg_set bg_set are set") - - def run_go(go_type): - go2fg = defaultdict(set) - go2bg = defaultdict(set) - - for _, row in res_go.iterrows(): - uid = str(row["query"]) - for term in row[f"{go_type}_terms"]: - go2bg[term].add(uid) - if uid in fg_set: - go2fg[term].add(uid) - - records = [] - N_fg = len(fg_set) - N_bg = len(bg_set) - - for term, fg_genes in go2fg.items(): - a = len(fg_genes) - if a == 0: - continue - b = N_fg - a - c = len(go2bg[term]) - a - d = N_bg - (a + b + c) - - _, p = fisher_exact([[a, b], [c, d]], alternative="greater") - records.append({ - "GO_Term": term, - "Count": a, - "GeneRatio": f"{a}/{N_fg}", - "p_value": p, - }) - - df = pd.DataFrame(records) - if df.empty: - return None, None - - df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) - df = df.sort_values("p_value").head(20) - - # βœ… Plotly Figure - fig = px.bar( - df, - x="-log10(p)", - y="GO_Term", - orientation="h", - title=f"GO Enrichment ({go_type})", - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - self.logger.log(f"βœ… Plotly Figure generated") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - fig.update_layout( - yaxis=dict(autorange="reversed"), - height=500, - margin=dict(l=10, r=10, t=40, b=10), - ) + self.logger.log("βœ… PercolatorAdapter complete") + + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": psm_filtered, + }, + tool_instance_name="IDFilter-strict" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-strict complete") + + # Build visualization cache for Filter results + for idxml_file in psm_filtered: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - return fig, df + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - go_results = {} + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - for go_type in ["BP", "CC", "MF"]: - fig, df_go = run_go(go_type) - if fig is not None: - go_results[go_type] = { - "fig": fig, - "df": df_go - } - self.logger.log(f"βœ… go_type generated") - - go_dir = results_dir / "go-terms" - go_dir.mkdir(parents=True, exist_ok=True) - - import json - go_data = {} - - for go_type in ["BP", "CC", "MF"]: - if go_type in go_results: - fig = go_results[go_type]["fig"] - df = go_results[go_type]["df"] - - go_data[go_type] = { - "fig_json": fig.to_json(), # Figure β†’ JSON string - "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts - } - - go_json_file = go_dir / "go_results.json" - with open(go_json_file, "w") as f: - json.dump(go_data, f) - st.session_state["go_results"] = go_results - st.session_state["go_ready"] = True if go_data else False - self.logger.log("βœ… GO enrichment analysis complete") - + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + # --- IDMapper --- + self.logger.log("πŸ—ΊοΈ Mapping IDs to isobaric consensus features...") + for iso, psm, mapped in zip(iso_consensus, psm_filtered, mapped_ids): + iso_stem = Path(iso).stem + with st.spinner(f"IDMapper ({iso_stem})"): + if not self.executor.run_topp( + "IDMapper", + { + "in": [iso], + "id": [psm], + "out": [mapped], + }, + tool_instance_name="IDMapper-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDMapper complete") + + # --- FileMerger --- + self.logger.log("πŸ”— Merging mapped consensus files...") + with st.spinner("FileMerger"): + if not self.executor.run_topp( + "FileMerger", + { + "in": mapped_ids, + "out": [merged_id], + }, + tool_instance_name="FileMerger-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… FileMerger complete") + + # --- ProteinInference --- + self.logger.log("🧩 Running protein inference...") + with st.spinner("ProteinInference"): + if not self.executor.run_topp( + "ProteinInference", + { + "in": [merged_id], + "out": [protein_id], + }, + tool_instance_name="ProteinInference-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinInference complete") + + # --- IDFilter-lenient (Protein) --- + self.logger.log("πŸ”¬ Filtering proteins...") + with st.spinner("IDFilter (Protein)"): + if not self.executor.run_topp( + "IDFilter", + { + "in": [protein_id], + "out": [protein_filter], + }, + tool_instance_name="IDFilter-lenient" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-lenient (Protein) complete") + + # ================================ + # ✨ NEW: 8️⃣ IDConflictResolver (protein_filter β†’ protein_resolved) + # ================================ + self.logger.log("βš–οΈ Resolving ID conflicts...") + with st.spinner("IDConflictResolver"): + if not self.executor.run_topp( + "IDConflictResolver", + { + "in": [protein_filter], + "out": [protein_resolved], + }, + tool_instance_name="IDConflictResolver-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDConflictResolver complete") + + # ================================ + # ✨ NEW: πŸ”Ÿ ProteinQuantifier (protein_resolved β†’ consensus_out) + # ================================ + self.logger.log("πŸ“ Running protein quantification...") + with st.spinner("ProteinQuantifier"): + if not self.executor.run_topp( + "ProteinQuantifier", + { + "in": [protein_resolved], + "out": [consensus_out], + }, + tool_instance_name="ProteinQuantifier-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinQuantifier complete") + self.logger.log("πŸ“„ Generating protein table...") + + self.logger.log("πŸŽ‰ WORKFLOW FINISHED") @st.fragment def results(self) -> None: diff --git a/src/common/results_helpers.py b/src/common/results_helpers.py index db3e103..2d38ad9 100644 --- a/src/common/results_helpers.py +++ b/src/common/results_helpers.py @@ -5,10 +5,8 @@ import numpy as np import streamlit as st from pathlib import Path -from scipy.stats import ttest_ind from pyopenms import IdXMLFile, MSExperiment, MzMLFile from src.workflow.ParameterManager import ParameterManager -from statsmodels.stats.multitest import multipletests def get_workflow_dir(workspace): """Get the workflow directory path.""" @@ -184,12 +182,15 @@ def build_spectra_cache(mzml_dir: Path, filename_to_index: dict) -> tuple[pl.Dat @st.cache_data -def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: - """Load CSV, compute stats (log2FC, p-value), build pivot_df and expr_df. +def load_abundance_data(workspace_path: str, csv_mtime: float, params_mtime: float = 0.0) -> tuple | None: + """Load CSV and build abundance matrices for downstream preprocessing. Args: workspace_path: Path to the workspace directory csv_mtime: Modification time of CSV file (used as cache key) + params_mtime: Modification time of params.json (used as cache key so + changing group assignments in Configure invalidates the cache + even when the CSV itself hasn't changed) Returns: Tuple of (pivot_df, expr_df, group_map) or None if data unavailable @@ -197,115 +198,166 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: workflow_dir = get_workflow_dir(Path(workspace_path)) quant_dir = workflow_dir / "results" / "quant_results" - if not quant_dir.exists(): - return None - - csv_files = sorted(quant_dir.glob("*.csv")) - if not csv_files: - return None - - csv_file = csv_files[0] - - try: - df = pd.read_csv(csv_file) - except Exception: - return None + parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - if df.empty: - return None + workflow_params = parameter_manager.get_parameters_from_json() + analysis_mode = workflow_params.get("analysis-mode", "LFQ") - # Get group mapping from parameters - param_manager = ParameterManager(workflow_dir) - params = param_manager.get_parameters_from_json() - group_map = { - key[11:]: value # Remove "mzML-group-" prefix - for key, value in params.items() - if key.startswith("mzML-group-") and value - } + if analysis_mode == "LFQ": + if not quant_dir.exists(): + return None - if not group_map: - return None + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None - df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) - df["Group"] = df["Reference"].map(group_map) - df = df.dropna(subset=["Group"]) + csv_file = csv_files[0] - groups = sorted(df["Group"].unique()) + try: + df = pd.read_csv(csv_file) + except Exception: + return None - if len(groups) < 2: - return None + if df.empty: + return None - group1, group2 = groups[:2] - - # Compute statistics per protein - stats_rows = [] - for protein, protein_df in df.groupby("ProteinName"): - g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values - g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values + # Get optional group mapping from parameters. + # Group information is not required at this stage; statistical testing + # happens in the Statistical page. + param_manager = ParameterManager(workflow_dir) + params = param_manager.get_parameters_from_json() + group_map = { + key[11:]: value # Remove "mzML-group-" prefix + for key, value in params.items() + if key.startswith("mzML-group-") and value + } - if len(g1_vals) < 2 or len(g2_vals) < 2: - pval = np.nan + df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) + + # Build sample display order. + if group_map: + sample_group_df = df[["Sample", "Reference"]].drop_duplicates() + sample_group_df["Group"] = sample_group_df["Reference"].map(group_map) + grouped_samples = [] + for grp in sorted(sample_group_df["Group"].dropna().unique()): + grouped_samples.extend( + sample_group_df[sample_group_df["Group"] == grp]["Sample"].tolist() + ) + remaining_samples = [ + s for s in sorted(df["Sample"].unique()) if s not in grouped_samples + ] + all_samples = grouped_samples + remaining_samples else: - _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False) - - mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan - mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan - - log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan + all_samples = sorted(df["Sample"].unique()) + + # Build pivot table + pivot_list = [] + for protein, group_df in df.groupby("ProteinName"): + peptides = ";".join(group_df["PeptideSequence"].unique()) + intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() + intensity_dict_complete = { + sample: intensity_dict.get(sample, 0) + for sample in all_samples + } + row = { + "ProteinName": protein, + **intensity_dict_complete, + "PeptideSequence": peptides, + } + pivot_list.append(row) + + pivot_df = pd.DataFrame(pivot_list) + pivot_df = pivot_df[["ProteinName"] + all_samples + ["PeptideSequence"]] + + # Build expression matrix (log2-transformed) + expr_df = pivot_df.set_index("ProteinName")[all_samples] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() + + return pivot_df, expr_df, group_map + + else: + if not quant_dir.exists(): + return None + + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None + + csv_file = csv_files[0] + + try: + df = pd.read_csv(csv_file, sep="\t", comment="#", engine="python") + except Exception: + return None + + if df.empty: + return None + + # ratio column removal + df = df.loc[:, ~df.columns.str.contains('ratio', case=False)] + + # exclude_indices = st.session_state.get("tmt_exclude_indices", []) + # group_map = st.session_state.get("tmt_group_map", {}) + # Get group mapping from parameters + parameter_manager = ParameterManager(Path(workflow_dir), "TOPP Workflow") + params = parameter_manager.get_parameters_from_json() + group_map = {} + for key, value in params.items(): + if key.startswith("TMT-group-") and value: + # Extract the numeric part from keys like "TMT-group-sample1" + match = re.search(r'sample(\d+)', key) + if match: + # Subtract 1 to convert to a 0-based index (0, 1, 2...). + # If your samples are already 0-based, remove the -1 adjustment. + index = str(int(match.group(1)) - 1) + group_map[index] = value + + # 1. Extract keys labeled as "skip" from group_map as integer list + exclude_indices = [ + int(k) for k, v in group_map.items() if v.lower() == "skip" + ] + + # 2. Remove "skip" entries from group_map (keep only actual group info) + group_map = { + int(k): v for k, v in group_map.items() if v.lower() != "skip" + } - stats_rows.append({ - "ProteinName": protein, - "log2FC": log2fc, - "p-value": pval, - }) + start_column_offset = 4 - stats_df = pd.DataFrame(stats_rows) + # st.write("exclude_indices:", exclude_indices) + # st.write("group_map:", group_map) - if not stats_df.empty: - mask = stats_df["p-value"].notna() - if mask.any(): - _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh") - stats_df.loc[mask, "p-adj"] = p_adj + if exclude_indices: + # st.write("Current columns:", df.columns.tolist()) + # st.write("Number of columns:", len(df.columns)) + # st.write("Exclude indices:", exclude_indices) + # st.write("Offset:", start_column_offset) + cols_to_drop = [df.columns[i + start_column_offset] for i in exclude_indices] + df_cleaned = df.drop(columns=cols_to_drop) else: - stats_df["p-adj"] = np.nan - - # Order samples by group (group2 first, then group1) - sample_group_df = df[["Sample", "Group"]].drop_duplicates() - group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist() - group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist() - all_samples = group2_samples + group1_samples - - # Build pivot table - pivot_list = [] - for protein, group_df in df.groupby("ProteinName"): - peptides = ";".join(group_df["PeptideSequence"].unique()) - intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() - intensity_dict_complete = { - sample: intensity_dict.get(sample, 0) - for sample in all_samples - } - row = { - "ProteinName": protein, - **intensity_dict_complete, - "PeptideSequence": peptides, - } - pivot_list.append(row) + df_cleaned = df.copy() + + current_cols = df_cleaned.columns.tolist() + sample_cols = current_cols[start_column_offset:] - pivot_df = pd.DataFrame(pivot_list) - pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left") - pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]] + # Ensure sample columns are numeric for downstream preprocessing/statistics. + pivot_df = df_cleaned.copy() + if sample_cols: + pivot_df[sample_cols] = pivot_df[sample_cols].apply(pd.to_numeric, errors='coerce') - # Build expression matrix (log2-transformed) - expr_df = pivot_df.set_index("ProteinName")[all_samples] - expr_df = expr_df.replace(0, np.nan) - expr_df = np.log2(expr_df + 1) - expr_df = expr_df.dropna() + protein_col = pivot_df.columns[0] + expr_df = pivot_df.set_index(protein_col)[sample_cols] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() - return pivot_df, expr_df, group_map + return pivot_df, expr_df, group_map def get_abundance_data(workspace: Path) -> tuple | None: - """Wrapper that handles cache key (workspace + CSV mtime). + """Wrapper that handles cache key (workspace + CSV mtime + params mtime). Args: workspace: Path to the workspace directory @@ -324,4 +376,49 @@ def get_abundance_data(workspace: Path) -> tuple | None: return None csv_mtime = csv_files[0].stat().st_mtime - return load_abundance_data(str(workspace), csv_mtime) + + params_file = workflow_dir / "params.json" + params_mtime = params_file.stat().st_mtime if params_file.exists() else 0.0 + + return load_abundance_data(str(workspace), csv_mtime, params_mtime) + + +def get_id_column(workspace: Path, pivot_df: pd.DataFrame) -> str: + """Resolve the protein/row identifier column for the active analysis mode. + + LFQ reports always use "ProteinName"; TMT reports use whatever the + report's first column is actually named (e.g. "protein"). + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + return "ProteinName" if analysis_mode == "LFQ" else pivot_df.columns[0] + + +def get_sample_group_map(workspace: Path, pivot_df: pd.DataFrame, group_map: dict) -> dict: + """Normalize group_map into {actual_sample_column_name: group_name}. + + LFQ group_map keys are already clean sample names (optionally with a + ".mzML" suffix). TMT group_map keys are 0-based channel indices that must + be matched against the report's actual "sampleN[...]" column names. + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + + if analysis_mode == "LFQ": + return { + k[:-5] if k.endswith(".mzML") else k: v + for k, v in group_map.items() + } + + actual_sample_names = pivot_df.columns.tolist() + norm_map = {} + for k, v in group_map.items(): + try: + sample_idx = int(k) + 1 + except (TypeError, ValueError): + continue + target_substring = f"sample{sample_idx}[" + real_full_name = next((name for name in actual_sample_names if target_substring in name), None) + if real_full_name: + norm_map[real_full_name] = v if v and v.strip() else "Unassigned" + return norm_map From 9c1df80a7386684f97800e40250ba153f1ba8392 Mon Sep 17 00:00:00 2001 From: Yoo HoJun Date: Thu, 16 Jul 2026 15:15:14 +0900 Subject: [PATCH 06/10] Add LFQ/TMT workflow mode split and downstream analysis pages Split WorkflowTest configure() into separate LFQ and TMT tool tabs, and add new preprocessing/analysis pages (filtering, normalization, imputation, statistical testing, GO enrichment, clustered heatmap, pathway analysis) built on openms_insight engine functions. --- content/enrichment.py | 141 ++ content/filtering.py | 173 +++ content/imputation.py | 145 ++ content/normalization.py | 242 ++++ content/results_abundance.py | 138 +- content/results_heatmap.py | 79 +- content/results_heatmap_clustered.py | 107 ++ content/results_pathway_analysis.py | 258 ++++ content/results_pca.py | 177 ++- content/results_proteomicslfq.py | 5 +- content/results_volcano.py | 91 +- content/statistical.py | 165 +++ requirements.txt | 8 +- src/WorkflowTest.py | 1841 +++++++++++++++++--------- src/common/results_helpers.py | 287 ++-- 15 files changed, 2978 insertions(+), 879 deletions(-) create mode 100644 content/enrichment.py create mode 100644 content/filtering.py create mode 100644 content/imputation.py create mode 100644 content/normalization.py create mode 100644 content/results_heatmap_clustered.py create mode 100644 content/results_pathway_analysis.py create mode 100644 content/statistical.py diff --git a/content/enrichment.py b/content/enrichment.py new file mode 100644 index 0000000..62fc9fa --- /dev/null +++ b/content/enrichment.py @@ -0,0 +1,141 @@ +"""Pathway Analysis Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column +# Import GO Enrichment modules from openms_insight engine +from openms_insight.analysis.enrichment import calculate_go_enrichment + +params = page_setup() +st.title("GO Enrichment Analysis") + +st.markdown( + """ +Identify overrepresented biological themes (BP, CC, MF) within your differentially expressed protein features using MyGene.info and Fisher's Exact Test. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# --- STEP 1: Upstream Statistics Checkpoint --- +if ( + "statistics_df" in st.session_state + and st.session_state["statistics_df"] is not None +): + final_statistics_report = st.session_state["statistics_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using analyzed matrices from the **Statistical Inference** step." + ) +else: + st.warning( + "⚠️ **Missing Prerequisites**: Statistical inference data not detected. Please run hypothesis testing first." + ) + st.page_link( + "content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬" + ) + st.stop() + +# --- STEP 2: Preprocessing Mapping Key Configuration --- +# Identify target identifier columns dynamically +abundance_result = get_abundance_data(st.session_state["workspace"]) +id_col = get_id_column(st.session_state["workspace"], abundance_result[0]) if abundance_result else "ProteinName" +if id_col not in final_statistics_report.columns: + st.error(f"❌ Structural Error: Column '{id_col}' is missing from the active matrix context.") + st.stop() + +# --- SECTION 1: Parameter Setup & Dynamic Cutoff Labels --- +st.subheader("Configure Enrichment Thresholds") + +# Check if target p-value should be adjusted or raw based on previous selections (Fallback safely to 'p-adj') +target_p_col = "p-adj" if "p-adj" in final_statistics_report.columns else "p-value" +p_label = ( + "Adjusted P-value (p-adj) Cutoff" + if target_p_col == "p-adj" + else "Raw P-value (p-value) Cutoff" +) + +ui_go_col1, ui_go_col2 = st.columns(2) + +with ui_go_col1: + p_cutoff = st.number_input( + f"πŸ”¬ {p_label}", + min_value=0.0001, + max_value=1.0, + value=0.05, + step=0.01, + format="%.4f", + help="Proteins with significance metrics below this value are mapped to the foreground cohort.", + ) + +with ui_go_col2: + fc_cutoff = st.number_input( + "πŸ“ˆ Absolute Difference Cutoff (|log2FC|)", + min_value=0.0, + max_value=10.0, + value=1.0, + step=0.1, + format="%.2f", + help="Proteins with absolute log2 fold change greater than or equal to this threshold will be selected.", + ) + +# --- SECTION 2: Execution and Interactive View Charts --- +st.markdown("
", unsafe_allow_html=True) +if st.button("πŸš€ Run GO Enrichment Analysis", type="primary", key="run_go_analysis"): + + with st.spinner("Querying MyGene.info API & executing hyper-geometric calculation loops..."): + # Convert internal pandas DataFrame to openms_insight Polars DataFrame expectation + stats_pl = pl.from_pandas(final_statistics_report) + + status, output = calculate_go_enrichment( + final_report=stats_pl, + id_col=id_col, + target_p_col=target_p_col, + p_cutoff=p_cutoff, + fc_cutoff=fc_cutoff, + ) + + # Route response structures based on analysis output status code + if status == "empty_data": + st.error("❌ No valid statistical rows found containing standard columns to run GO alignment.") + + elif status == "insufficient_proteins": + st.warning( + f"⚠️ Not enough significant proteins found to construct target datasets. " + f"(Criteria: {target_p_col} < {p_cutoff:.4f}, |log2FC| β‰₯ {fc_cutoff:.2f})." + ) + st.info(f"πŸ’‘ Found significant proteins count: **{output}**. Try relaxing your p-value or log2FC filters.") + + elif status == "success": + st.success("β­• GO Enrichment Analysis completed successfully!") + + # Display operational matrix scale + st.markdown( + f"πŸ“Š **Analysis Profile Scope**: Mapped **{output['fg_count']}** significant foreground profiles out of **{output['bg_count']}** reference background items." + ) + + # Build multi-tab interface layer for ontology subcategories + tabs = st.tabs([ + "🧬 Biological Process (BP)", + "πŸ”¬ Cellular Component (CC)", + "πŸ§ͺ Molecular Function (MF)" + ]) + categories_data = output["categories"] + + for idx, go_type in enumerate(["BP", "CC", "MF"]): + with tabs[idx]: + fig = categories_data[go_type]["fig"] + df_go = categories_data[go_type]["df"] + + if fig is not None and df_go is not None: + # Render plotly bar figures generated straight from backend engine + st.plotly_chart(fig, use_container_width=True) + + st.subheader(f"πŸ“Š {go_type} Results Dataframe") + st.dataframe(df_go, use_container_width=True) + else: + st.info(f"No statistically overrepresented terms identified for Category: **{go_type}**") \ No newline at end of file diff --git a/content/filtering.py b/content/filtering.py new file mode 100644 index 0000000..2bad00c --- /dev/null +++ b/content/filtering.py @@ -0,0 +1,173 @@ +"""Filtering Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import filtering functions from openms_insight package +from openms_insight.analysis.filter import ( + filter_low_abundance, + filter_low_repeatability, + filter_low_variance, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Filtering") + +st.markdown( + """ +Filter out low-quality proteins from your dataset based on abundance, repeatability, or variance thresholds. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Identify actual sample columns dynamically +sample_cols = [ + c + for c in pivot_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Original Data View --- +st.subheader("Original Abundance Table") +st.markdown( + f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering." +) +st.dataframe(pivot_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Filter Configuration --- +st.subheader("Configure Filter Engine") + +# Prepare Polars Metadata DataFrame required by openms_insight functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for filtering strategy +filter_method = st.selectbox( + "Select Filtering Method", + options=["Low Abundance", "Low Repeatability", "Low Variance"], + index=0, + help="Choose the statistical criteria to prune unreliable protein entries.", +) + +# Render threshold sliders dynamically based on the selected filter method +if filter_method == "Low Abundance": + st.markdown( + "**Low Abundance Filter**: Keeps rows where at least one group's median is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +elif filter_method == "Low Repeatability": + st.markdown( + "**Low Repeatability Filter**: Keeps rows where at least one group has a missing value ratio within the allowed maximum." + ) + threshold = st.slider( + "Max Missing Ratio", + min_value=0.0, + max_value=100.0, + value=50.0, + step=5.0, + help="Allowed missing value (zero or null) ratio per group.", + ) + +elif filter_method == "Low Variance": + st.markdown( + "**Low Variance Filter**: Keeps rows where at least one group's variance is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +# --- SECTION 3: Filter Execution and Collected Results View --- +if st.button("Apply Filter", type="primary"): + # Convert the original Pandas DataFrame into a Polars LazyFrame graph + quant_lazy = pl.from_pandas(pivot_df).lazy() + + # Route execution to the chosen openms_insight engine function + if filter_method == "Low Abundance": + filtered_lazy = filter_low_abundance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + elif filter_method == "Low Repeatability": + # Convert percent slider input to ratio expected by the function (e.g., 50.0% -> 0.5) + filtered_lazy = filter_low_repeatability( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + max_missing_ratio=threshold / 100.0, + ) + elif filter_method == "Low Variance": + filtered_lazy = filter_low_variance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + + # Collect the evaluated lazy graph and convert back to Pandas for visualization + filtered_df = strip_stat_columns(filtered_lazy.collect().to_pandas()) + st.session_state["filtered_df"] = filtered_df + + # Layout response metrics and the filtered matrix + st.success(f"Successfully applied **{filter_method}** filter!") + + # Display dataset scale compression stats + col1, col2, col3 = st.columns(3) + col1.metric("Original Proteins", pivot_df.shape[0]) + col2.metric("Filtered Proteins", filtered_df.shape[0]) + col3.metric( + "Removed Proteins", pivot_df.shape[0] - filtered_df.shape[0], delta=None + ) + + st.subheader("Filtered Abundance Table") + if filtered_df.empty: + st.warning( + "The filtered table is empty. Try relaxing the threshold constraints." + ) + else: + st.dataframe(filtered_df, use_container_width=True) \ No newline at end of file diff --git a/content/imputation.py b/content/imputation.py new file mode 100644 index 0000000..4350263 --- /dev/null +++ b/content/imputation.py @@ -0,0 +1,145 @@ +"""Imputation Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import imputation algorithms from openms_insight engine +from openms_insight.analysis.imputation import impute_mar, impute_smallest_value + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Missing Value Imputation") + +st.markdown( + """ +Handle missing values (zeros or nulls) in your quantification matrix using biological group-aware (MAR) or absolute lowest limit (MNAR) techniques. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load base dataset and clean dictionary keys +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Pipeline Checkpoint: Fetch upstream filtered data if available, fallback to raw pivot matrix +if "filtered_df" in st.session_state and st.session_state["filtered_df"] is not None: + base_df = strip_stat_columns(st.session_state["filtered_df"]) + st.session_state["filtered_df"] = base_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No filtering history found. Operating on the original unfiltered table." + ) + +# 2. Identify actual sample columns dynamically based on the current active matrix +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Input Matrix Summary --- +st.subheader("Input Matrix Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Imputation Configuration --- +st.subheader("Configure Imputation Engine") + +# Build Polars structural metadata DataFrame +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for core missingness assumption strategy +impute_category = st.selectbox( + "Select Imputation Class", + options=["MAR (Missing At Random)", "MNAR (Missing Not At Random)"], + index=0, + help="MAR uses group metrics (Mean/Median). MNAR shifts values below the limit of detection.", +) + +# Render algorithmic options sub-menus based on the parent selection +if impute_category == "MAR (Missing At Random)": + st.markdown( + "**Group Character Imputation**: Fills missing metrics leveraging sample properties belonging to the same group." + ) + strategy_opt = st.radio( + "Mathematical Strategy", + options=["median", "mean"], + index=0, + horizontal=True, + ) + +elif impute_category == "MNAR (Missing Not At Random)": + st.markdown( + "**Smallest Value Imputation**: Replaces missing items with the minimum values detected to reflect technical dropout limits." + ) + scope_opt = st.radio( + "Detection Minimum Scope", + options=["row", "global"], + index=0, + horizontal=True, + help="'row' targets current protein minimum; 'global' searches the entire mass spectrometry matrix profile.", + ) + +# --- SECTION 3: Imputation Execution --- +if st.button("Apply Imputation", type="primary"): + # Initialize optimization pipeline graph via lazy loading conversion + quant_lazy = pl.from_pandas(base_df).lazy() + + # Route configuration matrix parameters to designated engine function channels + if impute_category == "MAR (Missing At Random)": + imputed_lazy = impute_mar( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + strategy=strategy_opt, + ) + elif impute_category == "MNAR (Missing Not At Random)": + imputed_lazy = impute_smallest_value( + quantification_data=quant_lazy, metadata=metadata_pl, scope=scope_opt + ) + + # Resolve lazy graph optimization tree and push to display data frame structure + imputed_df = strip_stat_columns(imputed_lazy.collect().to_pandas()) + + # πŸ’Ύ Save current output into Session State for down-stream processing (Normalization, Statistics) + st.session_state["imputed_df"] = imputed_df + + st.success(f"Successfully finalized **{impute_category}** imputation step!") + + # Calculate and display a quick performance matrix check + st.subheader("Imputed Result Table") + st.dataframe(imputed_df, use_container_width=True) \ No newline at end of file diff --git a/content/normalization.py b/content/normalization.py new file mode 100644 index 0000000..c0e97e8 --- /dev/null +++ b/content/normalization.py @@ -0,0 +1,242 @@ +"""Normalization Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import normalization engine functions from openms_insight +from openms_insight.analysis.normalization import ( + normalize_samples, + scale_data, + transform_data, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None: + """Keep preprocessing tables intensity-only before statistical analysis.""" + if df is None: + return None + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Normalization & Scaling") + +st.markdown( + """ +Standardize and transform your protein abundance profiles to correct for technical variations and optimize statistical distributions. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +filtered_df = strip_stat_columns(st.session_state.get("filtered_df")) +imputed_df = strip_stat_columns(st.session_state.get("imputed_df")) +normalized_df = strip_stat_columns(st.session_state.get("normalized_df")) +if filtered_df is not None: + st.session_state["filtered_df"] = filtered_df +if imputed_df is not None: + st.session_state["imputed_df"] = imputed_df +if normalized_df is not None: + st.session_state["normalized_df"] = normalized_df + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = imputed_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = filtered_df + st.warning( + "⚠️ **Imputation Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original unfiltered table." + ) + +# 2. Extract actual active sample columns dynamically +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("### Pipeline Overview") +st.caption("Data flows in order: Filtering -> Imputation -> Normalization") + +step_rows = [ + { + "Step": "Filtering", + "Status": "Done" if filtered_df is not None else "Not run", + "Rows": filtered_df.shape[0] if filtered_df is not None else "-", + "Cols": filtered_df.shape[1] if filtered_df is not None else "-", + }, + { + "Step": "Imputation", + "Status": "Done" if imputed_df is not None else "Not run", + "Rows": imputed_df.shape[0] if imputed_df is not None else "-", + "Cols": imputed_df.shape[1] if imputed_df is not None else "-", + }, + { + "Step": "Normalization", + "Status": "Done" if normalized_df is not None else "Not run", + "Rows": normalized_df.shape[0] if normalized_df is not None else "-", + "Cols": normalized_df.shape[1] if normalized_df is not None else "-", + }, +] +st.dataframe(pd.DataFrame(step_rows), hide_index=True, use_container_width=True) + +with st.expander("Show step tables", expanded=False): + if filtered_df is not None: + st.markdown("#### Filtering output") + st.dataframe(filtered_df.head(10), use_container_width=True) + if imputed_df is not None: + st.markdown("#### Imputation output") + st.dataframe(imputed_df.head(10), use_container_width=True) + if normalized_df is not None: + st.markdown("#### Normalization output") + st.dataframe(normalized_df.head(10), use_container_width=True) + if filtered_df is None and imputed_df is None and normalized_df is None: + st.info("No preprocessing outputs yet. Start from Filtering.") + +st.markdown("---") + +# --- SECTION 2: Normalization Parameter Configuration --- +st.subheader("Configure Preprocessing & Scaling Chains") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2, col3 = st.columns(3) + +with col1: + st.markdown("### 🧬 1. Mathematical Transformation") + transform_strategy = st.selectbox( + "Select Transformation", + options=["None", "log2", "log10", "square_root", "cube_root"], + index=0, + help="Compress data dynamic range and stabilize heteroscedastic variance profiles.", + ) + +with col2: + st.markdown("### πŸ§ͺ 2. Sample Normalization") + norm_strategy = st.selectbox( + "Select Normalization", + options=["None", "sum", "median", "pqn", "reference_feature", "quantile"], + index=0, + help="Perform column-wise corrections to account for variable sample loading concentrations.", + ) + + # Conditionally display target input field for reference feature matching + ref_feature_input = None + if norm_strategy == "reference_feature": + ref_feature_input = st.text_input( + "Reference Protein Name (ID)", + value="", + placeholder="e.g., P01234 or GAPDH", + help=f"Enter the exact unique identifier string matching a key inside the '{id_col}' column.", + ) + +with col3: + st.markdown("### πŸ“Š 3. Row Scaling") + scaling_strategy = st.selectbox( + "Select Scaling Mode", + options=["None", "mean_centering", "auto_scaling", "pareto_scaling", "range_scaling"], + index=0, + help="Adjust individual feature weights to make low and high abundance proteins comparable.", + ) + + +# --- SECTION 3: Normalization Pipe Sequential Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Apply Normalization Pipelines", type="primary"): + + # Validate reference feature selection if active before hitting polars execution layers + if norm_strategy == "reference_feature" and not ref_feature_input: + st.error( + "❌ Validation Error: Please provide a valid Reference Protein Name to use the 'reference_feature' strategy." + ) + st.stop() + + # Convert pandas memory buffer into optimization lazy dataframe tree graph + processing_lazy = pl.from_pandas(base_df).lazy() + + # Execute Chain 1: Transform Matrix Data + try: + processing_lazy = transform_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=transform_strategy, + ) + + # Execute Chain 2: Normalize Sample Intensities (Columns) + processing_lazy = normalize_samples( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=norm_strategy, + id_col=id_col, + reference_feature=ref_feature_input if norm_strategy == "reference_feature" else None, + ) + + # Execute Chain 3: Scale Individual Features (Rows) + processing_lazy = scale_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=scaling_strategy, + ) + + # Finalize and collect pipeline query graph optimizations + normalized_df = strip_stat_columns(processing_lazy.collect().to_pandas()) + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (Statistics Block) + st.session_state["normalized_df"] = normalized_df + + st.success("Successfully executed all selected normalization pipelines!") + + # Display the finalized transformation matrix view + st.subheader("Normalized Abundance Table") + st.dataframe(normalized_df, use_container_width=True) + + except ValueError as val_err: + # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein) + st.error(f"Engine Configuration Error: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/content/results_abundance.py b/content/results_abundance.py index a7ff453..38c42bc 100644 --- a/content/results_abundance.py +++ b/content/results_abundance.py @@ -1,9 +1,11 @@ """Abundance (ProteomicsLFQ) Results Page.""" import streamlit as st import pandas as pd +import numpy as np from pathlib import Path from src.common.common import page_setup from src.common.results_helpers import get_workflow_dir, get_abundance_data +from src.workflow.ParameterManager import ParameterManager params = page_setup() st.title("Abundance Quantification") @@ -11,7 +13,7 @@ st.markdown( """ View protein and PSM-level quantification from **ProteomicsLFQ**. -This page calculates differential expression statistics between sample groups. +This page focuses on raw abundance intensity for preprocessing. """ ) @@ -21,6 +23,10 @@ workflow_dir = get_workflow_dir(st.session_state["workspace"]) quant_dir = workflow_dir / "results" / "quant_results" +parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") + +workflow_params = parameter_manager.get_parameters_from_json() +analysis_mode = workflow_params.get("analysis-mode", "LFQ") if not quant_dir.exists(): st.info("No quantification results available yet. Please run the workflow first.") @@ -35,6 +41,55 @@ csv_file = csv_files[0] +def render_protein_table(pivot_df, is_lfq=True): + """Common function to render the protein-level abundance table""" + pivot_df = pivot_df.copy() + st.markdown("### Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples." + ) + + if is_lfq: + # Handle LFQ mode columns (Raw Intensity) + id_col = "ProteinName" + exclude_cols = [id_col, "PeptideSequence"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + display_cols = [id_col, "Intensity"] + sample_cols + ["PeptideSequence"] + help_text = "Raw sample intensities" + y_min = None + else: + # Handle non-LFQ mode columns (Log2-transformed Intensity) + id_col = "protein" + exclude_cols = [id_col, "n_proteins", "n_peptides", "protein_score"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols and "ratio" not in c.lower()] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply( + lambda row: [np.log2(v + 1) for v in row], axis=1 + ) + display_cols = [id_col, "Intensity"] + sample_cols + help_text = "Sample intensities (log2 scale)" + y_min = 0 + + # Filter to available columns, then sort and display + available_cols = [c for c in display_cols if c in pivot_df.columns] + view_df = pivot_df[available_cols] + + st.dataframe( + view_df, + column_config={ + "Intensity": st.column_config.BarChartColumn( + "Intensity", + help=help_text, + width="small", + y_min=y_min, + ), + }, + use_container_width=True, + ) + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) try: @@ -44,68 +99,53 @@ st.info("No data found in this file.") st.stop() - with protein_tab: - st.markdown("### Protein-Level Abundance Table") + result = get_abundance_data(st.session_state["workspace"]) - st.info( - "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." - ) + if analysis_mode == "LFQ": + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) - result = get_abundance_data(st.session_state["workspace"]) - if result is None: - st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.") - st.page_link("content/workflow_configure.py", label="Go to Configure", icon="βš™οΈ") - st.stop() + with protein_tab: + if result is None: + st.warning("Could not load abundance data. Please run the workflow first.") + st.stop() + + pivot_df, expr_df, group_map = result + render_protein_table(pivot_df, is_lfq=True) - pivot_df, expr_df, group_map = result + with psm_tab: + st.markdown("### PSM-level Quantification Table") + st.info( + "This table shows the PSM-level quantification data, including protein IDs, " + "peptide sequences, charge states, and intensities across samples. " + "Each row represents one peptide-spectrum match detected from the MS/MS analysis." + ) + st.dataframe(df, use_container_width=True) - # Display group comparison info - groups = sorted(set(group_map.values())) - if len(groups) >= 2: - group1, group2 = sorted(groups)[:2] - st.info(f"Statistical comparison: **{group2} vs {group1}**") + else: + pre_processing_tab, protein_tab = st.tabs(["Pre-processing", "Protein Table"]) - # Get sample columns (between stats and PeptideSequence) - sample_cols = [c for c in pivot_df.columns if c not in ["ProteinName", "log2FC", "p-value", "PeptideSequence"]] + if result is None: + st.info("πŸ’‘ Please run the workflow first to see results.") + st.stop() - pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + pivot_df, expr_df, group_map = result - # Reorder columns: place Intensity after p-value - display_cols = ["ProteinName", "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"] - display_df = pivot_df[display_cols] - - st.dataframe( - display_df.sort_values("p-value"), - column_config={ - "Intensity": st.column_config.BarChartColumn( - "Intensity", - help="Raw sample intensities", - width="small", - ), - }, - use_container_width=True, - ) + with pre_processing_tab: + st.write("### Final Results (Intensity matrix)") + st.dataframe(pivot_df.head(10)) - with psm_tab: - st.markdown("### PSM-level Quantification Table") - st.info( - "This table shows the PSM-level quantification data, including protein IDs, " - "peptide sequences, charge states, and intensities across samples. " - "Each row represents one peptide-spectrum match detected from the MS/MS analysis." - ) - st.dataframe(df, use_container_width=True) + with protein_tab: + render_protein_table(pivot_df, is_lfq=False) except Exception as e: st.error(f"Failed to load {csv_file.name}: {e}") st.markdown("---") -st.markdown("**Next steps:** Explore statistical visualizations") +st.markdown("**Next steps:** Continue preprocessing, then run statistical inference") col1, col2, col3 = st.columns(3) with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") + st.page_link("content/filtering.py", label="Filtering", icon="🧹") with col2: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") + st.page_link("content/imputation.py", label="Imputation", icon="🧩") with col3: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") + st.page_link("content/statistical.py", label="Statistical Inference", icon="πŸ”¬") \ No newline at end of file diff --git a/content/results_heatmap.py b/content/results_heatmap.py index 4ece3f4..104bff6 100644 --- a/content/results_heatmap.py +++ b/content/results_heatmap.py @@ -1,19 +1,18 @@ """Heatmap Results Page.""" import streamlit as st import numpy as np -import plotly.express as px -from scipy.cluster.hierarchy import linkage, leaves_list -from scipy.spatial.distance import pdist +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import Heatmap params = page_setup() st.title("Heatmap") st.markdown( """ -Hierarchically clustered heatmap of protein-level abundance (Z-score normalized). -Proteins and samples are ordered by similarity. +Interactive hierarchically clustered heatmap of protein-level abundance (Z-score normalized). +Powered by OpenMS-Insight multi-resolution engine. """ ) @@ -28,42 +27,68 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) -top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +sample_cols = expr_df.columns.tolist() +# UI settings (number of top variance proteins) +top_n = st.slider("Number of proteins (Highest Variance)", 20, 200, 50, key="heatmap_top_n") + +# Process data (variance selection -> Z-score normalization) var_series = expr_df.var(axis=1) top_proteins = var_series.sort_values(ascending=False).head(top_n).index heatmap_df = expr_df.loc[top_proteins] + +# Compute Z-scores and clean missing/invalid values heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) + # Melt and convert data to Polars to satisfy OpenMS-Insight component requirements + # Restore the id column from the index as a regular column + heatmap_z_reset = heatmap_z.reset_index() - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) + # Unpivot the wide-format matrix into long-format (X, Y, Intensity) + melted_df = heatmap_z_reset.melt( + id_vars=[id_col], + value_vars=sample_cols, + var_name="Sample", + value_name="Z_score" + ) - heatmap_clustered = heatmap_z.iloc[row_order, col_order] + # Add sample group mapping if available for heatmap categories + if sample_group_map: + melted_df["Group"] = melted_df["Sample"].map(sample_group_map) - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) + # Pack the Pandas DataFrame into a Polars LazyFrame + heatmap_pl_lazy = pl.from_pandas(melted_df).lazy() - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} + # Initialize the OpenMS-Insight Heatmap component and map attributes + heatmap_component = Heatmap( + cache_id="quantms_protein_heatmap", + x_column="Sample", + y_column=id_col, + data=heatmap_pl_lazy, + intensity_column="Z_score", + title="Protein Abundance Heatmap (Z-score)", + x_label="Samples", + y_label="Proteins", + colorscale="RdBu", + reversescale=True, + log_scale=False, # Z-score can be negative, so log scale must stay off + intensity_label="Z-score", + category_column=None, + min_points=10000, # Generous point-count ceiling so the full grid renders ) - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, use_container_width=True) + # Render the component + state_manager = st.session_state.get("state") + heatmap_component(state_manager=state_manager) else: st.warning("Insufficient data to generate the heatmap.") diff --git a/content/results_heatmap_clustered.py b/content/results_heatmap_clustered.py new file mode 100644 index 0000000..7104c3a --- /dev/null +++ b/content/results_heatmap_clustered.py @@ -0,0 +1,107 @@ +"""Clustered Heatmap Results Page.""" +import streamlit as st +import numpy as np +import polars as pl +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import ClusteredHeatmap + +params = page_setup() +st.title("Clustered Heatmap") + +st.markdown( + """ +A real grid heatmap (rows = proteins, columns = samples) with hierarchical +clustering dendrograms on both axes and a sample-group color bar, powered +by OpenMS-Insight. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +top_n = st.slider("Number of proteins (Highest Variance)", 10, 200, 30, key="clustered_heatmap_top_n") + +var_series = expr_df.var(axis=1) +top_proteins = var_series.sort_values(ascending=False).head(top_n).index +heatmap_df = expr_df.loc[top_proteins] + +heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) +heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() + +if heatmap_z.empty: + st.warning("Insufficient data to generate the heatmap.") + st.stop() + +heatmap_z_reset = heatmap_z.reset_index() +heatmap_lazy = pl.from_pandas(heatmap_z_reset).lazy() + +sample_cols = expr_df.columns.tolist() +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, +) + +# Assign group annotation-bar colors in sorted-group order (matching how +# ClusteredHeatmap._preprocess() orders unique groups internally). +group_palette = [ + "#00BFC4", # teal + "#F8766D", # salmon + "#7CAE00", # yellow-green + "#C77CFF", # lavender purple + "#E7B800", # gold/amber + "#619CFF", # blue + "#FF61C3", # pink/magenta + "#00BA38", # green + "#FF8C42", # orange + "#00B0F6", # sky blue +] +unique_groups = sorted(set(sample_group_map.values())) +group_colors = {g: group_palette[i % len(group_palette)] for i, g in enumerate(unique_groups)} + +heatmap_component = ClusteredHeatmap( + cache_id="quantms_clustered_heatmap", + cache_path=str(st.session_state["workspace"]), + id_col=id_col, + data=heatmap_lazy, + metadata=metadata_pl, + row_cluster=True, + col_cluster=True, + title="Protein Abundance Heatmap (Z-score, clustered)", + x_label="Samples", + y_label="Proteins", + colorscale=[[0, "#6699E0"], [0.5, "#FFFFFF"], [1, "#E06666"]], + reversescale=False, + intensity_label="Z-score", + group_colors=group_colors, +) + +state_manager = st.session_state.get("state") +# Scale height with the number of proteins so row labels stay readable - +# BaseComponent otherwise defaults to a flat 400px, too short for a +# dendrogram+heatmap composite with more than a handful of rows. +heatmap_height = max(600, min(1400, 300 + top_n * 20)) +heatmap_component(state_manager=state_manager, height=heatmap_height) + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") +with col2: + st.page_link("content/results_heatmap.py", label="Heatmap (original)", icon="πŸ”₯") diff --git a/content/results_pathway_analysis.py b/content/results_pathway_analysis.py new file mode 100644 index 0000000..f5eb4c1 --- /dev/null +++ b/content/results_pathway_analysis.py @@ -0,0 +1,258 @@ +import json +import mygene +import streamlit as st +import pandas as pd +import numpy as np +import plotly.express as px +import plotly.io as pio +from collections import defaultdict +from scipy.stats import fisher_exact +from pathlib import Path +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data + +# ================================ +# Page setup +# ================================ +params = page_setup() +st.title("ProteomicsLFQ Results") + +# ================================ +# Workspace check +# ================================ +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# ================================ +# _run_go_enrichment function +# ================================ +def _run_go_enrichment(pivot_df: pd.DataFrame, results_dir: Path): + p_cutoff = 0.05 + fc_cutoff = 1.0 + + analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() + + if analysis_df.empty: + st.error("No valid statistical data found for GO enrichment.") + st.write("❗ analysis_df is empty") + else: + with st.spinner("Fetching GO terms from MyGene.info API..."): + mg = mygene.MyGeneInfo() + + def get_clean_uniprot(name): + parts = str(name).split("|") + return parts[1] if len(parts) >= 2 else parts[0] + + analysis_df["UniProt"] = analysis_df["protein"].apply(get_clean_uniprot) + + bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() + fg_ids = analysis_df[ + (analysis_df["p-value"] < p_cutoff) & + (analysis_df["log2FC"].abs() >= fc_cutoff) + ]["UniProt"].dropna().astype(str).unique().tolist() + # st.write("βœ… get_clean_uniprot applied") + + if len(fg_ids) < 3: + st.warning( + f"Not enough significant proteins " + f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " + f"Found: {len(fg_ids)}" + ) + st.write("❗ Not enough significant proteins") + else: + res_list = mg.querymany( + bg_ids, scopes="uniprot", fields="go", as_dataframe=False + ) + res_go = pd.DataFrame(res_list) + if "notfound" in res_go.columns: + res_go = res_go[res_go["notfound"] != True] + + def extract_go_terms(go_data, go_type): + if not isinstance(go_data, dict) or go_type not in go_data: + return [] + terms = go_data[go_type] + if isinstance(terms, dict): + terms = [terms] + return list({t.get("term") for t in terms if "term" in t}) + + for go_type in ["BP", "CC", "MF"]: + res_go[f"{go_type}_terms"] = res_go["go"].apply( + lambda x: extract_go_terms(x, go_type) + ) + + annotated_ids = set(res_go["query"].astype(str)) + fg_set = annotated_ids.intersection(fg_ids) + bg_set = annotated_ids + # st.write(f"βœ… fg_set bg_set are set") + + def run_go(go_type): + go2fg = defaultdict(set) + go2bg = defaultdict(set) + + for _, row in res_go.iterrows(): + uid = str(row["query"]) + for term in row[f"{go_type}_terms"]: + go2bg[term].add(uid) + if uid in fg_set: + go2fg[term].add(uid) + + records = [] + N_fg = len(fg_set) + N_bg = len(bg_set) + + for term, fg_genes in go2fg.items(): + a = len(fg_genes) + if a == 0: + continue + b = N_fg - a + c = len(go2bg[term]) - a + d = N_bg - (a + b + c) + + _, p = fisher_exact([[a, b], [c, d]], alternative="greater") + records.append({ + "GO_Term": term, + "Count": a, + "GeneRatio": f"{a}/{N_fg}", + "p_value": p, + }) + + df = pd.DataFrame(records) + if df.empty: + return None, None + + df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) + df = df.sort_values("p_value").head(20) + + # βœ… Plotly Figure + fig = px.bar( + df, + x="-log10(p)", + y="GO_Term", + orientation="h", + title=f"GO Enrichment ({go_type})", + ) + + # st.write(f"βœ… Plotly Figure generated") + + fig.update_layout( + yaxis=dict(autorange="reversed"), + height=500, + margin=dict(l=10, r=10, t=40, b=10), + ) + + return fig, df + + go_results = {} + + for go_type in ["BP", "CC", "MF"]: + fig, df_go = run_go(go_type) + if fig is not None: + go_results[go_type] = { + "fig": fig, + "df": df_go + } + # st.write(f"βœ… go_type generated") + + go_dir = results_dir / "go-terms" + go_dir.mkdir(parents=True, exist_ok=True) + + go_data = {} + + for go_type in ["BP", "CC", "MF"]: + if go_type in go_results: + fig = go_results[go_type]["fig"] + df = go_results[go_type]["df"] + + go_data[go_type] = { + "fig_json": fig.to_json(), # Figure β†’ JSON string + "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts + } + + go_json_file = go_dir / "go_results.json" + with open(go_json_file, "w") as f: + json.dump(go_data, f) + st.session_state["go_results"] = go_results + st.session_state["go_ready"] = True if go_data else False + # st.write("βœ… GO enrichment analysis complete") + +# ================================ +# Load abundance data +# ================================ +results_dir = Path(st.session_state["workspace"]) / "topp-workflow" / "results" / "quant_results" +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result + +go_json_file = results_dir / "go-terms" / "go_results.json" + +go_input_df = pivot_df.copy() +if "ProteinName" in go_input_df.columns: + go_input_df = go_input_df.rename(columns={"ProteinName": "protein"}) + +_run_go_enrichment(go_input_df, results_dir) + +# ================================ +# Tabs +# ================================ +protein_tab, = st.tabs(["🧬 Protein Table"]) + +# ================================ +# Protein-level results +# ================================ +with protein_tab: + st.markdown("### 🧬 Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples.\n\n" + "Additionally, log2 fold change and p-values are calculated between sample groups." + ) + + if pivot_df.empty: + st.info("No protein-level data available.") + else: + st.session_state["pivot_df"] = pivot_df + st.dataframe(pivot_df.sort_values("p-value"), width="stretch") + +# ====================================================== +# GO Enrichment Results +# ====================================================== +st.markdown("---") +st.subheader("🧬 GO Enrichment Analysis") + +if not go_json_file.exists(): + st.info("GO Enrichment results are not available yet. Please run the analysis first.") +else: + with open(go_json_file, "r") as f: + go_data = json.load(f) + + bp_tab, cc_tab, mf_tab = st.tabs([ + "🧬 Biological Process", + "🏠 Cellular Component", + "βš™οΈ Molecular Function", + ]) + + for tab, go_type in zip([bp_tab, cc_tab, mf_tab], ["BP", "CC", "MF"]): + with tab: + if go_type not in go_data: + st.info(f"No enriched {go_type} terms found.") + continue + + fig_json = go_data[go_type]["fig_json"] + df_dict = go_data[go_type]["df_dict"] + + fig = pio.from_json(fig_json) + + df_go = pd.DataFrame(df_dict) + + if df_go.empty: + st.info(f"No enriched {go_type} terms found.") + else: + st.plotly_chart(fig, width="stretch") + + st.markdown(f"#### {go_type} Enrichment Results") + st.dataframe(df_go, width="stretch") \ No newline at end of file diff --git a/content/results_pca.py b/content/results_pca.py index 45ea8eb..29f441a 100644 --- a/content/results_pca.py +++ b/content/results_pca.py @@ -1,11 +1,10 @@ """PCA Results Page.""" -import streamlit as st import pandas as pd -import plotly.express as px -from sklearn.decomposition import PCA -from sklearn.preprocessing import StandardScaler +import polars as pl +import streamlit as st from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import PCAPlot params = page_setup() st.title("PCA Analysis") @@ -13,7 +12,7 @@ st.markdown( """ Principal Component Analysis (PCA) of protein-level abundance. -Samples are colored by group assignment to visualize clustering. +Samples are projected onto their principal components and colored by group assignment to visualize clustering. """ ) @@ -21,6 +20,7 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Load abundance data (base wide-format table + sample -> group mapping) result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -28,60 +28,143 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +# Mirrors statistical.py: PCA should run on the most-processed data available. +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-adj", "stat", "p-value"] +] +unique_groups = sorted({sample_group_map[s] for s in sample_cols if s in sample_group_map}) + +if len(sample_cols) < 2: + st.info("PCA requires at least 2 samples.") + st.stop() -top_n = 500 +if len(unique_groups) < 2: + st.warning( + "Only one biological group was detected - points will still be plotted, " + "but group-based coloring requires 2 or more groups." + ) -top_proteins = ( - pivot_df - .dropna(subset=["p-adj"]) - .sort_values("p-adj", ascending=True) - .head(top_n)["ProteinName"] +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples " + f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})." ) +st.dataframe(base_df, use_container_width=True) -expr_df_pca = expr_df.loc[ - expr_df.index.intersection(top_proteins) -] +st.markdown("---") + +# --- SECTION 2: PCA Configuration --- +st.subheader("Configure PCA") + +expr_df_wide = base_df.set_index(id_col)[sample_cols] +max_available = expr_df_wide.shape[0] + +if max_available <= 20: + top_n = max_available + st.caption(f"Using all {top_n} proteins for PCA (dataset too small for variance filtering).") +else: + top_n = st.slider( + "Number of proteins (Highest Variance)", + min_value=20, + max_value=min(5000, max_available), + value=min(500, max_available), + step=10, + key="pca_top_n", + help=( + "PCA is computed only on the N proteins with the highest variance " + "across samples, to reduce noise from low-variance/uninformative features." + ), + ) + +top_proteins = expr_df_wide.var(axis=1).sort_values(ascending=False).head(top_n).index +expr_df_pca = expr_df_wide.loc[top_proteins].reset_index() if expr_df_pca.shape[0] < 2: - st.info("Not enough proteins after p-value filtering for PCA.") + st.info("Not enough proteins after variance filtering for PCA.") st.stop() -X = expr_df_pca.T -X_scaled = StandardScaler().fit_transform(X) - -pca = PCA(n_components=2) -pcs = pca.fit_transform(X_scaled) - -pca_df = pd.DataFrame( - pcs, - columns=["PC1", "PC2"], - index=X.index +# Prepare structural Polars metadata DataFrame required by PCAPlot +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, ) +pca_lazy = pl.from_pandas(expr_df_pca).lazy() + +# 3. Initialize the OpenMS-Insight PCAPlot component (computes PCA internally) +try: + pca_component = PCAPlot( + cache_id="quantms_pca_plot", + data=pca_lazy, + metadata=metadata_pl, + sample_id_field="sample_id", + group_field="group", + n_components=5, + title="Sample PCA", + ) +except ValueError as e: + st.error(f"PCA computation failed: {e}") + st.stop() -norm_map = { - k.replace(".mzML", ""): v - for k, v in group_map.items() -} -pca_df["Group"] = pca_df.index.map(norm_map) - -fig_pca = px.scatter( - pca_df, - x="PC1", - y="PC2", - color="Group", - text=pca_df.index, -) +variance_ratio = pca_component.get_variance_ratio() +pc_columns = pca_component.get_pc_columns() -fig_pca.update_traces(textposition="top center") -fig_pca.update_layout( - xaxis_title=f"PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)", - yaxis_title=f"PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)", - height=600, -) +# 4. Let the user pick which component pair to view (no recomputation needed) +col1, col2 = st.columns(2) +with col1: + pc_x_label = st.selectbox("X-axis component", pc_columns, index=0, key="pca_pc_x") +with col2: + default_y_index = 1 if len(pc_columns) > 1 else 0 + pc_y_label = st.selectbox("Y-axis component", pc_columns, index=default_y_index, key="pca_pc_y") + +pc_x = int(pc_x_label.replace("PC", "")) +pc_y = int(pc_y_label.replace("PC", "")) -st.plotly_chart(fig_pca, use_container_width=True) +# 5. Render the component +state_manager = st.session_state.get("state") +pca_component(state_manager=state_manager, pc_x=pc_x, pc_y=pc_y, height=600) -st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") +st.markdown( + "**Explained variance:** " + + ", ".join(f"{col} {ratio * 100:.1f}%" for col, ratio in zip(pc_columns, variance_ratio)) +) +st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by variance)") st.markdown("---") st.markdown("**Other visualizations:**") diff --git a/content/results_proteomicslfq.py b/content/results_proteomicslfq.py index 77eb332..fde2ab9 100644 --- a/content/results_proteomicslfq.py +++ b/content/results_proteomicslfq.py @@ -45,15 +45,14 @@ st.markdown("### 🧬 Protein-Level Abundance Table") st.info( "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." + "same protein and aggregating their intensities across samples." ) if pivot_df.empty: st.info("No protein-level data available.") else: st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df.sort_values("p-value"), use_container_width=True) + st.dataframe(pivot_df, use_container_width=True) # ====================================================== # GO Enrichment Results diff --git a/content/results_volcano.py b/content/results_volcano.py index 8502489..db2702f 100644 --- a/content/results_volcano.py +++ b/content/results_volcano.py @@ -1,9 +1,9 @@ """Volcano Plot Results Page.""" import streamlit as st -import plotly.express as px -import numpy as np +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column +from openms_insight import VolcanoPlot params = page_setup() st.title("Volcano Plot") @@ -19,6 +19,19 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Check if statistical analysis results are available in the session state +if "statistics_df" not in st.session_state or st.session_state["statistics_df"] is None: + st.info("Statistical analysis data not found. Please run the statistical engine first.") + st.page_link("content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬") + st.stop() + +# Retrieve the completed statistical analysis DataFrame +statistics_df = st.session_state["statistics_df"] + +if statistics_df.empty: + st.info("No data available for volcano plot.") + st.stop() + result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -26,16 +39,13 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) -if pivot_df.empty: - st.info("No data available for volcano plot.") - st.stop() - -volcano_df = pivot_df.copy() -volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"]) - -volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) +# 2. Clean data and convert to Polars for component input +volcano_df = statistics_df.dropna(subset=["log2FC", "p-adj"]).copy() +volcano_pl_lazy = pl.from_pandas(volcano_df).lazy() +# 3. Configure UI sliders (changing thresholds does not invalidate cache) fc_thresh = st.slider( "log2 Fold Change threshold", min_value=0.5, @@ -52,49 +62,34 @@ step=0.001, ) -volcano_df["Significance"] = "Not significant" -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", -] = "Up-regulated" - -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", -] = "Down-regulated" - -fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["ProteinName", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } +# 4. Initialize the OpenMS-Insight VolcanoPlot component +volcano_plot_component = VolcanoPlot( + cache_id="quantms_volcano_plot", + data=volcano_pl_lazy, + log2fc_column="log2FC", + pvalue_column="p-adj", + label_column=id_col, + up_color="#E74C3C", + down_color="#3498DB", + ns_color="#95A5A6", + show_threshold_lines=True, + threshold_line_style="dash", ) -fig_volcano.add_vline(x=fc_thresh, line_dash="dash") -fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") -fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - -# Make x-axis symmetric around zero -max_abs_fc = volcano_df["log2FC"].abs().max() -x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding +# 5. Render the component +state_manager = st.session_state.get("state") # Inject the project state management object -fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, +volcano_plot_component( + state_manager=state_manager, + fc_threshold=fc_thresh, + p_threshold=p_thresh, + max_labels=10, # Display labels for the top N significant proteins height=600, ) -st.plotly_chart(fig_volcano, use_container_width=True) - -up_count = (volcano_df["Significance"] == "Up-regulated").sum() -down_count = (volcano_df["Significance"] == "Down-regulated").sum() +# 6. Keep the existing statistical summary and bottom links +up_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh)).sum() +down_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh)).sum() st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") st.markdown("---") diff --git a/content/statistical.py b/content/statistical.py new file mode 100644 index 0000000..2e2a46d --- /dev/null +++ b/content/statistical.py @@ -0,0 +1,165 @@ +"""Statistical Inference Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import statistics engine functions from openms_insight +from openms_insight.analysis.statistics import calculate_statistical_tests, adjust_fdr_lazy + +params = page_setup() +st.title("Statistical Inference") + +st.markdown( + """ +Run differential expression analysis to identify statistically significant proteins across your biological groups. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract actual active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] +unique_groups = sorted(list(set([sample_group_map[s] for s in sample_cols if s in sample_group_map]))) +group_count = len(unique_groups) + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Dynamic Statistical Parameter Configuration --- +st.subheader("Configure Statistical Engine") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2 = st.columns(2) + +with col1: + st.markdown("### πŸ”¬ 1. Hypothesis Testing Method") + + # Route available method options dynamically based on the group count + if group_count == 2: + method_options = ["limma_like", "welch", "paired"] + help_text = "'limma_like' uses Empirical Bayes variance shrinking. 'welch' is for unequal variances. 'paired' is for dependent samples." + elif group_count >= 3: + method_options = ["limma_like", "anova"] + help_text = "'limma_like' supports multi-group design matrices. 'anova' computes standard row-wise One-way ANOVA." + else: + st.error("❌ Statistical testing requires at least 2 unique sample groups.") + st.stop() + + selected_method = st.selectbox( + "Select Statistical Test", + options=method_options, + index=0, + help=help_text + ) + +with col2: + st.markdown("### πŸ›‘οΈ 2. Multiple Testing Correction (FDR)") + selected_fdr = st.selectbox( + "Select FDR Adjustment Strategy", + options=["BH", "Bonferroni", "None"], + index=0, + help="'BH' (Benjamini-Hochberg) controls False Discovery Rate. 'Bonferroni' is strict Family-Wise Error Rate control." + ) + +# --- SECTION 3: Statistical Query Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Run Statistical Analysis", type="primary"): + + # Convert active pandas dataframe into polars lazyframe graph + stats_lazy = pl.from_pandas(base_df).lazy() + + try: + # Execute Chain 1: Calculate core statistics (Adds log2FC, stat, p-value) + stats_lazy = calculate_statistical_tests( + quantification_data=stats_lazy, + metadata=metadata_pl, + method=selected_method + ) + + # Execute Chain 2: Adjust Multiple Testing (Adds p-adj) + stats_lazy = adjust_fdr_lazy( + quantification_data=stats_lazy, + strategy=selected_fdr + ) + + # Resolve lazy graph optimization tree and bring back to pandas memory + statistics_df = stats_lazy.collect().to_pandas() + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (e.g., Volcano plot, Volcano/Heatmap UI) + st.session_state["statistics_df"] = statistics_df + + st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!") + + # Display the finalized statistics table view + st.subheader("Statistical Analysis Results") + st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`") + st.dataframe(statistics_df, use_container_width=True) + + except ValueError as val_err: + st.error(f"Engine Validation Fallure: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index aac2879..5d20693 100644 --- a/requirements.txt +++ b/requirements.txt @@ -142,6 +142,11 @@ scipy scikit-learn openms-insight>=0.1.13 polars>=1.0.0 +# Forces polars to prefer its most CPU-compatible native runtime. +# Without this, polars can select an AVX-optimized runtime (e.g. runtime-32) +# that access-violation crashes (0xC0000005 in _polars_runtime.pyd) on some +# CPUs (seen on AMD Threadripper PRO 3995WX on Windows) during dataframe ops. +polars-runtime-compat cython easypqp>=0.1.34 pyprophet>=2.2.0 @@ -149,4 +154,5 @@ mygene # Redis Queue dependencies (for online mode) redis>=5.0.0 rq>=1.16.0 -statsmodels \ No newline at end of file +statsmodels +polars \ No newline at end of file diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index 2af9bda..120efb0 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -1,9 +1,10 @@ import streamlit as st from pathlib import Path +import re import pandas as pd import plotly.express as px -#from streamlit_plotly_events import plotly_events -#from pyopenms import IdXMLFile +from streamlit_plotly_events import plotly_events +from pyopenms import IdXMLFile from scipy.stats import ttest_ind import numpy as np import mygene @@ -14,7 +15,7 @@ from src.common.common import page_setup from src.common.results_helpers import get_abundance_data from src.common.results_helpers import parse_idxml, build_spectra_cache -#from openms_insight import Table, Heatmap, LinePlot, SequenceView +from openms_insight import Table, Heatmap, LinePlot, SequenceView # params = page_setup() class WorkflowTest(WorkflowManager): @@ -47,6 +48,29 @@ def configure(self) -> None: self.ui.select_input_file("mzML-files", multiple=True, reactive=True) self.ui.select_input_file("fasta-file", multiple=False) + self.params = self.parameter_manager.get_parameters_from_json() + saved_mode = self.params.get("analysis-mode", "LFQ") + + self.ui.input_widget( + key="analysis-mode", + default=saved_mode, + name="Analysis Mode", + widget_type="selectbox", + options=["LFQ", "TMT"], + help="Choose between Label-Free Quantification (LFQ) or Tandem Mass Tag (TMT) analysis.", + reactive=True + ) + + self.params = self.parameter_manager.get_parameters_from_json() + current_mode = self.params.get("analysis-mode", "LFQ") + + if current_mode == "LFQ": + self.render_lfq_tabs() + else: + self.render_tmt_tabs() + + def render_lfq_tabs(self): + st.subheader("LFQ Analysis Mode") t = st.tabs(["**Identification**", "**Rescoring**", "**Filtering**", "**Library Generation**", "**Quantification**", "**Group Selection**"]) with t[0]: @@ -70,10 +94,10 @@ def configure(self) -> None: st.info(""" **Decoy Database Settings:** * **method**: How decoy sequences are generated from target protein sequences. - *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly - rearranges the amino acids. Both methods preserve the amino acid composition - of the original protein, ensuring decoys have similar properties to real sequences - for accurate false discovery rate (FDR) estimation. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. """) self.ui.input_TOPP( "DecoyDatabase", @@ -102,7 +126,7 @@ def configure(self) -> None: st.info(comet_info) comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] if not self.params.get("generate-decoys", True): # Only show decoy_string when not generating decoys comet_include.append("PeptideIndexing:decoy_string") @@ -111,7 +135,7 @@ def configure(self) -> None: "CometAdapter", custom_defaults={ "threads": 8, - "instrument": "high_res", + "instrument": "low_res", "missed_cleavages": 2, "min_peptide_length": 6, "max_peptide_length": 40, @@ -120,17 +144,19 @@ def configure(self) -> None: "isotope_error": "0/1", "precursor_charge": "2:4", "precursor_mass_tolerance": 20.0, - "fragment_mass_tolerance": 0.02, - "fragment_bin_offset": 0.0, + "fragment_mass_tolerance": 0.6, + "fragment_bin_offset": 0.4, "max_variable_mods_in_peptide": 3, "minimum_peaks": 1, "clip_nterm_methionine": "true", - "PeptideIndexing:IL_equivalent": "true", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)", + "PeptideIndexing:IL_equivalent": True, "PeptideIndexing:unmatched_action": "warn", "PeptideIndexing:decoy_string": "rev_", + "mass_recalibration": False, }, - flag_parameters=["PeptideIndexing:IL_equivalent"], include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "mass_recalibration"], exclude_parameters=["second_enzyme"], ) @@ -151,10 +177,10 @@ def configure(self) -> None: "subset_max_train": 300000, "decoy_pattern": "rev_", "score_type": "pep", - "post_processing_tdc": "true", + "post_processing_tdc": True, }, - flag_parameters=["post_processing_tdc"], include_parameters=percolator_include, + flag_parameters=["post_processing_tdc"], exclude_parameters=["out_type"], ) @@ -250,6 +276,10 @@ def configure(self) -> None: "psmFDR": 0.01, "proteinFDR": 0.01, "picked_proteinFDR": "true", + "alignment_order": "star", + "protein_quantification": "unique_peptides", + "quantification_method": "feature_intensity", + "protein_inference": "aggregation", }, include_parameters=["intThreshold", "psmFDR", "proteinFDR"], ) @@ -300,6 +330,294 @@ def configure(self) -> None: if orphaned_keys: self.parameter_manager.save_parameters() + def render_tmt_tabs(self): + st.subheader("TMT Analysis Mode") + # Create tabs for different analysis steps. + t = st.tabs( + ["**IsobaricAnalyzer**", "**CometAdapter**", "**PercolatorAdapter**", "**IDFilter**", "**IDMapper**", "**FileMerger**", + "**ProteinInference**", "**IDFilter**", "**IDConflictResolver**", "**ProteinQuantifier**", "**Group Selection**"] + ) + with t[0]: + # Checkbox for decoy generation + # reactive=True ensures the parent configure() fragment re-runs when checkbox changes, + # so conditional UI (DecoyDatabase settings) updates immediately + self.ui.input_widget( + key="generate-decoys", + default=True, + name="Generate Decoy Database", + widget_type="checkbox", + help="Generate reversed decoy sequences for FDR calculation. Disable if your FASTA already contains decoys.", + reactive=True, + ) + + # Reload params to get current checkbox value after it was saved + self.params = self.parameter_manager.get_parameters_from_json() + + # Show DecoyDatabase settings if generating decoys + if self.params.get("generate-decoys", True): + st.info(""" + **Decoy Database Settings:** + * **method**: How decoy sequences are generated from target protein sequences. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. + """) + self.ui.input_TOPP( + "DecoyDatabase", + custom_defaults={ + "decoy_string": "rev_", + "decoy_string_position": "prefix", + "method": "reverse", + }, + include_parameters=["method"], + ) + + comet_info = """ + **Identification (Comet):** + * **enzyme**: The enzyme used for peptide digestion. + * **missed_cleavages**: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. + * **fixed_modifications**: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **variable_modifications**: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **instrument**: Type of instrument (high_res or low_res). Use 'high_res' for high-resolution MS2 (Orbitrap, TOF), 'low_res' for ion trap. + * **fragment_mass_tolerance**: Fragment mass tolerance for MS2 matching. + * **fragment_bin_offset**: Offset for binning MS2 spectra. Typically 0.0 for high-res, 0.4 for low-res instruments. + """ + if not self.params.get("generate-decoys", True): + comet_info += """* **PeptideIndexing:decoy_string**: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions + in the protein database to indicate decoy proteins. + """ + st.info(comet_info) + + st.write(Path(self.workflow_dir, "results")) + + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + if not self.params.get("generate-decoys", True): + # Only show decoy_string when not generating decoys + comet_include.append("PeptideIndexing:decoy_string") + + self.ui.input_TOPP( + "IsobaricAnalyzer", + custom_defaults={ + "tmt11plex:reference_channel": 126, + "type": "tmt11plex", + "extraction:select_activation": "auto", + "extraction:reporter_mass_shift": 0.002, + "extraction:min_reporter_intensity": 0.0, + "extraction:min_precursor_purity": 0.0, + "extraction:precursor_isotope_deviation": 10.0, + "quantification:isotope_correction": "false", + }, + tool_instance_name="IsobaricAnalyzer-TMT", + reactive=True, + ) + with t[1]: + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset", "PeptideIndexing:IL_equivalent"] + self.ui.input_TOPP( + "CometAdapter", + custom_defaults={ + "PeptideIndexing:IL_equivalent": True, + "clip_nterm_methionine": "true", + "instrument": "high_res", + "missed_cleavages": 2, + "min_peptide_length": 6, + "max_peptide_length": 40, + "enzyme": "Trypsin/P", + "PeptideIndexing:unmatched_action": "warn", + "max_variable_mods_in_peptide": 3, + "precursor_mass_tolerance": 4.5, + "isotope_error": "0/1", + "precursor_error_units": "ppm", + "num_hits": 1, + "num_enzyme_termini": "fully", + "fragment_bin_offset": 0.0, + "minimum_peaks": 10, + "precursor_charge": "2:4", + "fragment_mass_tolerance": 0.015, + "PeptideIndexing:unmatched_action": "warn", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)\nTMT6plex (K)\nTMT6plex (N-term)", + "debug": 0, + "force": True, + }, + include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "force"], + exclude_parameters=["second_enzyme"], + tool_instance_name="CometAdapter-TMT", + ) + with t[2]: + st.info(""" + **Filtering (IDFilter):** + * **score:type_peptide**: Score used for filtering. If empty, the main score is used. + * **score:psm**: The score which should be reached by a peptide hit to be kept. (use 'NAN' to disable this filter) + """) + self.ui.input_TOPP( + "PercolatorAdapter", + custom_defaults={ + "subset_max_train": 300000, + "decoy_pattern": "DECOY_", + "score_type": "pep", + "post_processing_tdc": True, + "debug": 0, + }, + flag_parameters=["post_processing_tdc"], + tool_instance_name="PercolatorAdapter-TMT", + ) + + with t[3]: + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_peptide": "q-value", + "score:psm": 0.10, + }, + tool_instance_name="IDFilter-strict", + ) + with t[4]: + st.info(""" + **Quantification (ProteomicsLFQ):** + * **intThreshold**: Peak intensity threshold applied in seed detection. + * **psmFDR**: FDR threshold for sub-protein level (e.g. 0.05=5%). Use -FDR_type to choose the level. Cutoff is applied at the highest level. If Bayesian inference was chosen, it is equivalent with a peptide FDR + * **proteinFDR**: Protein FDR threshold (0.05=5%). + """) + self.ui.input_TOPP( + "IDMapper", + custom_defaults={ + "threads": 8, + "debug": 0, + }, + tool_instance_name="IDMapper-TMT", + ) + with t[5]: + self.ui.input_TOPP( + "FileMerger", + custom_defaults={ + "in_type": "consensusXML", + "append_method": "append_cols", + "annotate_file_origin": True, + "threads": 8, + }, + flag_parameters=["annotate_file_origin"], + tool_instance_name="FileMerger-TMT", + ) + with t[6]: + self.ui.input_TOPP( + "ProteinInference", + custom_defaults={ + "threads": 8, + "picked_decoy_string": "DECOY_", + "picked_fdr": "true", + "protein_fdr": "true", + "Algorithm:use_shared_peptides": "true", + "Algorithm:annotate_indistinguishable_groups": "true", + "Algorithm:score_type": "PEP", + "Algorithm:score_aggregation_method": "best", + "Algorithm:min_peptides_per_protein": 1, + }, + tool_instance_name="ProteinInference-TMT", + ) + with t[7]: + # A single checkbox widget for workflow logic. + # self.ui.input_widget("run-python-script", False, "Run custom Python script") * + # Generate input widgets for a custom Python tool, located at src/python-tools. + # Parameters are specified within the file in the DEFAULTS dictionary. + # self.ui.input_python("example") * + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_protein": "q-value", + "score:proteingroup": 0.01, + "score:psm": 0.01, + "delete_unreferenced_peptide_hits": True, + "remove_decoys": True + }, + flag_parameters=["delete_unreferenced_peptide_hits", "remove_decoys"], + tool_instance_name="IDFilter-lenient", + ) + with t[8]: + self.ui.input_TOPP( + "IDConflictResolver", + custom_defaults={ + "threads": 4, + }, + tool_instance_name="IDConflictResolver-TMT", + ) + + with t[9]: + self.ui.input_TOPP( + "ProteinQuantifier", + custom_defaults={ + "method": "top", + "top:N": 3, + "top:aggregate": "median", + "top:include_all": True, + "ratios": True, + "threads": 8, + "debug": 0, + }, + flag_parameters=["top:include_all", "ratios"], + tool_instance_name="ProteinQuantifier-TMT", + ) + with t[10]: + st.markdown("### πŸ§ͺ TMT Sample Group Assignment") + + latest_params = self.parameter_manager.get_parameters_from_json() + type_key = ( + f"{self.parameter_manager.topp_param_prefix}" + "IsobaricAnalyzer-TMT:1:type" + ) + selected_type = str( + st.session_state.get(type_key) + or latest_params.get("IsobaricAnalyzer-TMT", {}).get("type") + or "tmt11plex" + ).lower() + + m = re.search(r'\d+', selected_type) + is_supported_type = any(label in selected_type for label in ["tmt", "itraq"]) + if not m or not is_supported_type: + st.warning("Please select a supported isobaric type in the IsobaricAnalyzer tab first.") + else: + num_plex = int(m.group()) + channels = [f"sample{i+1}" for i in range(num_plex)] + st.caption(f"Isobaric type: **{selected_type}** - {num_plex} channels") + st.info("Assign a group name to each channel. Use **'skip'** to exclude a channel.") + + for row_start in range(0, num_plex, 2): + c1, c2 = st.columns(2) + + left_idx = row_start + left_channel = channels[left_idx] + with c1: + self.ui.input_widget( + key=f"TMT-group-{left_channel}", + default="", + name=f"Group for channel {left_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + right_idx = row_start + 1 + if right_idx < num_plex: + right_channel = channels[right_idx] + with c2: + self.ui.input_widget( + key=f"TMT-group-{right_channel}", + default="", + name=f"Group for channel {right_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + # Remove orphaned params from a previously selected larger plex + self.params = self.parameter_manager.get_parameters_from_json() + valid_keys = {f"TMT-group-{ch}" for ch in channels} + orphaned = [k for k in self.params if k.startswith("TMT-group-") and k not in valid_keys] + if orphaned: + for k in orphaned: + del self.params[k] + self.parameter_manager.save_parameters() + def execution(self) -> bool: """ Refactored TOPP workflow execution: @@ -352,639 +670,944 @@ def execution(self) -> bool: st.info(f"Using original FASTA: {fasta_path.name}") database_fasta = fasta_path - # ================================ - # 1️⃣ Directory setup - # ================================ - results_dir = Path(self.workflow_dir, "results") - comet_dir = results_dir / "comet_results" - perc_dir = results_dir / "percolator_results" - filter_dir = results_dir / "filter_results" - quant_dir = results_dir / "quant_results" - - for d in [comet_dir, perc_dir, filter_dir, quant_dir]: - d.mkdir(parents=True, exist_ok=True) - - self.logger.log("πŸ“ Output directories created") - - # # ================================ - # # 2️⃣ File path definitions (per sample) - # # ================================ - comet_results = [] - percolator_results = [] - filter_results = [] - - for mz in in_mzML: - stem = Path(mz).stem - comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) - percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) - filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + current_mode = self.params.get("analysis-mode", "LFQ") + st.write(f"Current analysis mode: **{current_mode}**") - # ================================ - # 3️⃣ Per-file processing - # ================================ - for i, mz in enumerate(in_mzML): - stem = Path(mz).stem - st.info(f"Processing sample: {stem}") + if current_mode == "LFQ": + self.logger.log("βš™οΈ Running LFQ workflow") - self.logger.log("πŸ”¬ Starting per-sample processing...") + # ================================ + # 1️⃣ Directory setup + # ================================ + results_dir = Path(self.workflow_dir, "results") + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + filter_dir = results_dir / "psm_filter" + quant_dir = results_dir / "quant_results" + + results_dir = Path(self.workflow_dir, "input-files") + + for d in [comet_dir, perc_dir, filter_dir, quant_dir]: + d.mkdir(parents=True, exist_ok=True) + + self.logger.log("πŸ“ Output directories created") + + # ================================ + # 2️⃣ File path definitions (per sample) + # ================================ + comet_results = [] + percolator_results = [] + filter_results = [] + + for mz in in_mzML: + stem = Path(mz).stem + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) + filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + + # ================================ + # 3️⃣ Per-file processing + # ================================ + for i, mz in enumerate(in_mzML): + stem = Path(mz).stem + st.info(f"Processing sample: {stem}") + + self.logger.log("πŸ”¬ Starting per-sample processing...") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - # --- CometAdapter --- - self.logger.log("πŸ”Ž Running peptide search...") - with st.spinner(f"CometAdapter ({stem})"): - comet_extra_params = {"database": str(database_fasta)} - if self.params.get("generate-decoys", True): - # Propagate decoy_string from DecoyDatabase - comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + ): + self.logger.log("Workflow stopped due to error") + return False - if not self.executor.run_topp( - "CometAdapter", - { - "in": in_mzML, - "out": comet_results, - }, - comet_extra_params, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Get fragment tolerance from CometAdapter parameters for visualization - comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") - frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) - frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" - - # Build visualization cache for Comet results - results_dir_path = Path(self.workflow_dir, "results") - cache_dir = results_dir_path / "insight_cache" - cache_dir.mkdir(parents=True, exist_ok=True) - - # Get mzML directory - mzml_dir = Path(in_mzML[0]).parent - - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} - - for idxml_file in comet_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - self.logger.log("βœ… Peptide search complete") + self.logger.log("βœ… Peptide search complete") - # --- PercolatorAdapter --- - self.logger.log("πŸ“Š Running rescoring...") - with st.spinner(f"PercolatorAdapter ({stem})"): - if not self.executor.run_topp( - "PercolatorAdapter", - { - "in": comet_results, - "out": percolator_results, - }, - {"decoy_pattern": decoy_string}, # Always propagated from upstream - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Percolator results - for idxml_file in percolator_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # if not Path(comet_results).exists(): + # st.error(f"CometAdapter failed for {stem}") + # st.stop() - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter ({stem})"): + if not self.executor.run_topp( + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + {"decoy_pattern": decoy_string}, # Always propagated from upstream + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - self.logger.log("βœ… Rescoring complete") + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # if not Path(percolator_results[i]).exists(): - # st.error(f"PercolatorAdapter failed for {stem}") - # st.stop() + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - # --- IDFilter --- - self.logger.log("πŸ”§ Filtering identifications...") - with st.spinner(f"IDFilter ({stem})"): - if not self.executor.run_topp( - "IDFilter", - { - "in": percolator_results, - "out": filter_results, - }, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Filter results - for idxml_file in filter_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + self.logger.log("βœ… Rescoring complete") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # if not Path(percolator_results[i]).exists(): + # st.error(f"PercolatorAdapter failed for {stem}") + # st.stop() - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter ({stem})"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": filter_results, + }, + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Build visualization cache for Filter results + for idxml_file in filter_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - self.logger.log("βœ… Filtering complete") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # if not Path(filter_results[i]).exists(): - # st.error(f"IDFilter failed for {stem}") - # st.stop() + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # ================================ - # EasyPQP Spectral Library Generation (optional) - # ================================ - if self.params.get("generate-library", False): - self.logger.log("πŸ“š Building spectral library with EasyPQP...") - st.info("Building spectral library with EasyPQP...") - library_dir = Path(self.workflow_dir, "results", "library") - library_dir.mkdir(parents=True, exist_ok=True) - - psms_files, peaks_files = [], [] - - for filter_idxml in filter_results: - original_stem = Path(filter_idxml).stem.replace("_filter", "") - matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) - if not matching_mzml: - self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") - continue - - # easypqp library requires specific extensions for file recognition: - # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension - # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension - # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name - psms_out = str(library_dir / f"{original_stem}.psmpkl") - peaks_out = str(library_dir / f"{original_stem}.peakpkl") - - convert_cmd = [ - "easypqp", "convert", - "--pepxml", filter_idxml, - "--spectra", matching_mzml, - "--psms", psms_out, - "--peaks", peaks_out - ] - if self.executor.run_command(convert_cmd): - psms_files.append(psms_out) - peaks_files.append(peaks_out) - - if psms_files: - # easypqp library outputs TSV format (despite common .pqp extension) - library_tsv = str(library_dir / "spectral_library.tsv") - library_cmd = ["easypqp", "library", "--out", library_tsv] - - if not self.params.get("library-use-fdr", False): - # --nofdr only skips FDR recalculation, NOT threshold filtering - # Set all thresholds to 1.0 to bypass filtering for pre-filtered input - library_cmd.extend([ - "--nofdr", - "--psm_fdr_threshold", "1.0", - "--peptide_fdr_threshold", "1.0", - "--protein_fdr_threshold", "1.0" - ]) - else: - # Apply user-specified FDR filtering - library_cmd.extend([ - "--psm_fdr_threshold", - str(self.params.get("library-psm-fdr", 0.01)), - "--peptide_fdr_threshold", - str(self.params.get("library-peptide-fdr", 0.01)), - "--protein_fdr_threshold", - str(self.params.get("library-protein-fdr", 0.01)) - ]) - - for psms, peaks in zip(psms_files, peaks_files): - library_cmd.extend([psms, peaks]) - - if self.executor.run_command(library_cmd): - self.logger.log("βœ… Spectral library created") - st.success("Spectral library created") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + self.logger.log("βœ… Filtering complete") + + # if not Path(filter_results[i]).exists(): + # st.error(f"IDFilter failed for {stem}") + # st.stop() + + # ================================ + # EasyPQP Spectral Library Generation (optional) + # ================================ + if self.params.get("generate-library", False): + self.logger.log("πŸ“š Building spectral library with EasyPQP...") + st.info("Building spectral library with EasyPQP...") + library_dir = Path(self.workflow_dir, "results", "library") + library_dir.mkdir(parents=True, exist_ok=True) + + psms_files, peaks_files = [], [] + + for filter_idxml in filter_results: + original_stem = Path(filter_idxml).stem.replace("_filter", "") + matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) + if not matching_mzml: + self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") + continue + + # easypqp library requires specific extensions for file recognition: + # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension + # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension + # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name + psms_out = str(library_dir / f"{original_stem}.psmpkl") + peaks_out = str(library_dir / f"{original_stem}.peakpkl") + + convert_cmd = [ + "easypqp", "convert", + "--pepxml", filter_idxml, + "--spectra", matching_mzml, + "--psms", psms_out, + "--peaks", peaks_out + ] + if self.executor.run_command(convert_cmd): + psms_files.append(psms_out) + peaks_files.append(peaks_out) + + if psms_files: + # easypqp library outputs TSV format (despite common .pqp extension) + library_tsv = str(library_dir / "spectral_library.tsv") + library_cmd = ["easypqp", "library", "--out", library_tsv] + + if not self.params.get("library-use-fdr", False): + # --nofdr only skips FDR recalculation, NOT threshold filtering + # Set all thresholds to 1.0 to bypass filtering for pre-filtered input + library_cmd.extend([ + "--nofdr", + "--psm_fdr_threshold", "1.0", + "--peptide_fdr_threshold", "1.0", + "--protein_fdr_threshold", "1.0" + ]) + else: + # Apply user-specified FDR filtering + library_cmd.extend([ + "--psm_fdr_threshold", + str(self.params.get("library-psm-fdr", 0.01)), + "--peptide_fdr_threshold", + str(self.params.get("library-peptide-fdr", 0.01)), + "--protein_fdr_threshold", + str(self.params.get("library-protein-fdr", 0.01)) + ]) + + for psms, peaks in zip(psms_files, peaks_files): + library_cmd.extend([psms, peaks]) + + if self.executor.run_command(library_cmd): + self.logger.log("βœ… Spectral library created") + st.success("Spectral library created") + else: + self.logger.log("Warning: Failed to build spectral library") else: - self.logger.log("Warning: Failed to build spectral library") - else: - self.logger.log("Warning: No PSMs converted for library generation") + self.logger.log("Warning: No PSMs converted for library generation") + + st.success(f"βœ“ {stem} identification completed") + + # # ================================ + # # 4️⃣ ProteomicsLFQ (cross-sample) + # # ================================ + self.logger.log("πŸ“ˆ Running cross-sample quantification...") + st.info("Running ProteomicsLFQ (cross-sample quantification)") + + quant_mztab = str(quant_dir / "openms_quant.mzTab") + quant_cxml = str(quant_dir / "openms.consensusXML") + quant_msstats = str(quant_dir / "openms_msstats.csv") + + with st.spinner("ProteomicsLFQ"): + combined_in = " ".join(in_mzML) + combined_ids = " ".join(filter_results) + self.logger.log(f"COMBINED_IN {combined_in}", 1) + self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) + self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) + self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + + # βœ… Streamlit output (debug view) + st.markdown("### πŸ” ProteomicsLFQ Input Debug") + st.write("**combined_in:**", combined_in) + st.write("**combined_in type:**", type(combined_in).__name__) + + st.write("**combined_ids:**", combined_ids) + st.write("**combined_ids type:**", type(combined_ids).__name__) + + if not self.executor.run_topp( + "ProteomicsLFQ", + { + "in": [in_mzML], + "ids": [filter_results], + "out": [quant_mztab], + "out_cxml": [quant_cxml], + "out_msstats": [quant_msstats], + }, + { + "fasta": str(database_fasta), + "threads": 12, + # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 + "PeptideQuantification:extract:IM_window": "0.0", + "PeptideQuantification:faims:merge_features": "false", + }, + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… Quantification complete") + + # if not Path(quant_mztab).exists(): + # st.error("ProteomicsLFQ failed: mzTab not created") + # st.stop() + + # ================================ + # 5️⃣ Final report + # # ================================ + st.success("πŸŽ‰ TOPP workflow completed successfully") + st.write("πŸ“ Results directory:") + st.code(str(results_dir)) + + st.write("πŸ“„ Generated files:") + st.write(f"- mzTab: {quant_mztab}") + st.write(f"- consensusXML: {quant_cxml}") + st.write(f"- MSstats CSV: {quant_msstats}") - st.success(f"βœ“ {stem} identification completed") + return True + else: + self.logger.log("βš™οΈ Running TMT workflow") - # ================================ - # 4️⃣ ProteomicsLFQ (cross-sample) - # ================================ - self.logger.log("πŸ“ˆ Running cross-sample quantification...") - st.info("Running ProteomicsLFQ (cross-sample quantification)") + results_dir = Path(self.workflow_dir, "results") + iso_dir = results_dir / "isobaric_consensusXML" + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + psm_filter_dir = results_dir / "psm_filter" + map_dir = results_dir / "idmapper" + merge_dir = results_dir / "merged" + protein_dir = results_dir / "protein" + msstats_dir = results_dir / "msstats" + quant_dir = results_dir / "quant_results" + + iso_consensus = [] + comet_results = [] + percolator_results = [] + psm_filtered = [] + mapped_ids = [] + + for d in [ + iso_dir, comet_dir, perc_dir, psm_filter_dir, + map_dir, merge_dir, protein_dir, msstats_dir, quant_dir + ]: + d.mkdir(parents=True, exist_ok=True) + + for mz in in_mzML: + stem = Path(mz).stem + iso_consensus.append(str(iso_dir / f"{stem}_iso.consensusXML")) + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_comet_perc.idXML")) + psm_filtered.append(str(psm_filter_dir / f"{stem}_comet_perc_filter.idXML")) + mapped_ids.append(str(map_dir / f"{stem}_comet_perc_filter_map.consensusXML")) + + merged_id = str(merge_dir / "ID_mapper_merge.consensusXML") + protein_id = str(protein_dir / "ID_mapper_merge_epi.consensusXML") + protein_filter = str(protein_dir / "ID_mapper_merge_epi_filter.consensusXML") + protein_resolved = str(protein_dir / "ID_mapper_merge_epi_filter_resconf.consensusXML") + consensus_out = str(quant_dir / "openms_design_protein_openms.csv") + + # --- IsobaricAnalyzer --- + self.logger.log("🏷️ Running isobaric labeling analysis...") + with st.spinner("IsobaricAnalyzer"): + if not self.executor.run_topp( + "IsobaricAnalyzer", + { + "in": in_mzML, + "out": iso_consensus, + }, + tool_instance_name="IsobaricAnalyzer-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IsobaricAnalyzer complete") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + tool_instance_name="CometAdapter-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… CometAdapter complete") + + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - quant_mztab = str(quant_dir / "openms_quant.mzTab") - quant_cxml = str(quant_dir / "openms.consensusXML") - quant_msstats = str(quant_dir / "openms_msstats.csv") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - with st.spinner("ProteomicsLFQ"): - combined_in = " ".join(in_mzML) - combined_ids = " ".join(filter_results) - self.logger.log(f"COMBINED_IN {combined_in}", 1) - self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) - self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) - self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # βœ… Streamlit output (debug view) - st.markdown("### πŸ” ProteomicsLFQ Input Debug") - st.write("**combined_in:**", combined_in) - st.write("**combined_in type:**", type(combined_in).__name__) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - st.write("**combined_ids:**", combined_ids) - st.write("**combined_ids type:**", type(combined_ids).__name__) + self.logger.log("βœ… Peptide search complete") + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter"): if not self.executor.run_topp( - "ProteomicsLFQ", - { - "in": [in_mzML], - "ids": [filter_results], - "out": [quant_mztab], - "out_cxml": [quant_cxml], - "out_msstats": [quant_msstats], - }, - { - "fasta": str(database_fasta), - "psmFDR": 0.5, - "proteinFDR": 0.5, - "threads": 12, - # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 - "PeptideQuantification:extract:IM_window": "0.0", - "PeptideQuantification:faims:merge_features": "false", - } - ): + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + tool_instance_name="PercolatorAdapter-TMT", + ): self.logger.log("Workflow stopped due to error") return False - self.logger.log("βœ… Quantification complete") - - # ====================================================== - # ⚠️ 5️⃣ GO Enrichment Analysis (INLINE IN EXECUTION) - # ====================================================== - workspace_path = Path(self.workflow_dir).parent - res = get_abundance_data(workspace_path) - if res is not None: - pivot_df, _, _ = res - self.logger.log("βœ… pivot_df loaded, starting GO enrichment...") - self._run_go_enrichment(pivot_df, results_dir) - else: - st.warning("GO enrichment skipped: abundance data not available.") + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # ================================ - # 5️⃣ Final report - # # ================================ - st.success("πŸŽ‰ TOPP workflow completed successfully") - st.write("πŸ“ Results directory:") - st.code(str(results_dir)) - - return True - - def _run_go_enrichment(self, pivot_df: pd.DataFrame, results_dir: Path): - p_cutoff = 0.05 - fc_cutoff = 1.0 - - analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() - - if analysis_df.empty: - st.error("No valid statistical data found for GO enrichment.") - self.logger.log("❗ analysis_df is empty") - else: - with st.spinner("Fetching GO terms from MyGene.info API..."): - mg = mygene.MyGeneInfo() - - def get_clean_uniprot(name): - parts = str(name).split("|") - return parts[1] if len(parts) >= 2 else parts[0] - - analysis_df["UniProt"] = analysis_df["ProteinName"].apply(get_clean_uniprot) - - bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() - fg_ids = analysis_df[ - (analysis_df["p-value"] < p_cutoff) & - (analysis_df["log2FC"].abs() >= fc_cutoff) - ]["UniProt"].dropna().astype(str).unique().tolist() - self.logger.log("βœ… get_clean_uniprot applied") - - if len(fg_ids) < 3: - st.warning( - f"Not enough significant proteins " - f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " - f"Found: {len(fg_ids)}" - ) - self.logger.log("❗ Not enough significant proteins") - else: - res_list = mg.querymany( - bg_ids, scopes="uniprot", fields="go", as_dataframe=False - ) - res_go = pd.DataFrame(res_list) - if "notfound" in res_go.columns: - res_go = res_go[res_go["notfound"] != True] - - def extract_go_terms(go_data, go_type): - if not isinstance(go_data, dict) or go_type not in go_data: - return [] - terms = go_data[go_type] - if isinstance(terms, dict): - terms = [terms] - return list({t.get("term") for t in terms if "term" in t}) - - for go_type in ["BP", "CC", "MF"]: - res_go[f"{go_type}_terms"] = res_go["go"].apply( - lambda x: extract_go_terms(x, go_type) - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - annotated_ids = set(res_go["query"].astype(str)) - fg_set = annotated_ids.intersection(fg_ids) - bg_set = annotated_ids - self.logger.log(f"βœ… fg_set bg_set are set") - - def run_go(go_type): - go2fg = defaultdict(set) - go2bg = defaultdict(set) - - for _, row in res_go.iterrows(): - uid = str(row["query"]) - for term in row[f"{go_type}_terms"]: - go2bg[term].add(uid) - if uid in fg_set: - go2fg[term].add(uid) - - records = [] - N_fg = len(fg_set) - N_bg = len(bg_set) - - for term, fg_genes in go2fg.items(): - a = len(fg_genes) - if a == 0: - continue - b = N_fg - a - c = len(go2bg[term]) - a - d = N_bg - (a + b + c) - - _, p = fisher_exact([[a, b], [c, d]], alternative="greater") - records.append({ - "GO_Term": term, - "Count": a, - "GeneRatio": f"{a}/{N_fg}", - "p_value": p, - }) - - df = pd.DataFrame(records) - if df.empty: - return None, None - - df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) - df = df.sort_values("p_value").head(20) - - # βœ… Plotly Figure - fig = px.bar( - df, - x="-log10(p)", - y="GO_Term", - orientation="h", - title=f"GO Enrichment ({go_type})", - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - self.logger.log(f"βœ… Plotly Figure generated") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - fig.update_layout( - yaxis=dict(autorange="reversed"), - height=500, - margin=dict(l=10, r=10, t=40, b=10), - ) + self.logger.log("βœ… PercolatorAdapter complete") + + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": psm_filtered, + }, + tool_instance_name="IDFilter-strict" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-strict complete") + + # Build visualization cache for Filter results + for idxml_file in psm_filtered: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - return fig, df + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - go_results = {} + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - for go_type in ["BP", "CC", "MF"]: - fig, df_go = run_go(go_type) - if fig is not None: - go_results[go_type] = { - "fig": fig, - "df": df_go - } - self.logger.log(f"βœ… go_type generated") - - go_dir = results_dir / "go-terms" - go_dir.mkdir(parents=True, exist_ok=True) - - import json - go_data = {} - - for go_type in ["BP", "CC", "MF"]: - if go_type in go_results: - fig = go_results[go_type]["fig"] - df = go_results[go_type]["df"] - - go_data[go_type] = { - "fig_json": fig.to_json(), # Figure β†’ JSON string - "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts - } - - go_json_file = go_dir / "go_results.json" - with open(go_json_file, "w") as f: - json.dump(go_data, f) - st.session_state["go_results"] = go_results - st.session_state["go_ready"] = True if go_data else False - self.logger.log("βœ… GO enrichment analysis complete") - + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + # --- IDMapper --- + self.logger.log("πŸ—ΊοΈ Mapping IDs to isobaric consensus features...") + for iso, psm, mapped in zip(iso_consensus, psm_filtered, mapped_ids): + iso_stem = Path(iso).stem + with st.spinner(f"IDMapper ({iso_stem})"): + if not self.executor.run_topp( + "IDMapper", + { + "in": [iso], + "id": [psm], + "out": [mapped], + }, + tool_instance_name="IDMapper-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDMapper complete") + + # --- FileMerger --- + self.logger.log("πŸ”— Merging mapped consensus files...") + with st.spinner("FileMerger"): + if not self.executor.run_topp( + "FileMerger", + { + "in": mapped_ids, + "out": [merged_id], + }, + tool_instance_name="FileMerger-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… FileMerger complete") + + # --- ProteinInference --- + self.logger.log("🧩 Running protein inference...") + with st.spinner("ProteinInference"): + if not self.executor.run_topp( + "ProteinInference", + { + "in": [merged_id], + "out": [protein_id], + }, + tool_instance_name="ProteinInference-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinInference complete") + + # --- IDFilter-lenient (Protein) --- + self.logger.log("πŸ”¬ Filtering proteins...") + with st.spinner("IDFilter (Protein)"): + if not self.executor.run_topp( + "IDFilter", + { + "in": [protein_id], + "out": [protein_filter], + }, + tool_instance_name="IDFilter-lenient" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-lenient (Protein) complete") + + # ================================ + # ✨ NEW: 8️⃣ IDConflictResolver (protein_filter β†’ protein_resolved) + # ================================ + self.logger.log("βš–οΈ Resolving ID conflicts...") + with st.spinner("IDConflictResolver"): + if not self.executor.run_topp( + "IDConflictResolver", + { + "in": [protein_filter], + "out": [protein_resolved], + }, + tool_instance_name="IDConflictResolver-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDConflictResolver complete") + + # ================================ + # ✨ NEW: πŸ”Ÿ ProteinQuantifier (protein_resolved β†’ consensus_out) + # ================================ + self.logger.log("πŸ“ Running protein quantification...") + with st.spinner("ProteinQuantifier"): + if not self.executor.run_topp( + "ProteinQuantifier", + { + "in": [protein_resolved], + "out": [consensus_out], + }, + tool_instance_name="ProteinQuantifier-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinQuantifier complete") + self.logger.log("πŸ“„ Generating protein table...") + + self.logger.log("πŸŽ‰ WORKFLOW FINISHED") @st.fragment def results(self) -> None: diff --git a/src/common/results_helpers.py b/src/common/results_helpers.py index db3e103..2d38ad9 100644 --- a/src/common/results_helpers.py +++ b/src/common/results_helpers.py @@ -5,10 +5,8 @@ import numpy as np import streamlit as st from pathlib import Path -from scipy.stats import ttest_ind from pyopenms import IdXMLFile, MSExperiment, MzMLFile from src.workflow.ParameterManager import ParameterManager -from statsmodels.stats.multitest import multipletests def get_workflow_dir(workspace): """Get the workflow directory path.""" @@ -184,12 +182,15 @@ def build_spectra_cache(mzml_dir: Path, filename_to_index: dict) -> tuple[pl.Dat @st.cache_data -def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: - """Load CSV, compute stats (log2FC, p-value), build pivot_df and expr_df. +def load_abundance_data(workspace_path: str, csv_mtime: float, params_mtime: float = 0.0) -> tuple | None: + """Load CSV and build abundance matrices for downstream preprocessing. Args: workspace_path: Path to the workspace directory csv_mtime: Modification time of CSV file (used as cache key) + params_mtime: Modification time of params.json (used as cache key so + changing group assignments in Configure invalidates the cache + even when the CSV itself hasn't changed) Returns: Tuple of (pivot_df, expr_df, group_map) or None if data unavailable @@ -197,115 +198,166 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: workflow_dir = get_workflow_dir(Path(workspace_path)) quant_dir = workflow_dir / "results" / "quant_results" - if not quant_dir.exists(): - return None - - csv_files = sorted(quant_dir.glob("*.csv")) - if not csv_files: - return None - - csv_file = csv_files[0] - - try: - df = pd.read_csv(csv_file) - except Exception: - return None + parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - if df.empty: - return None + workflow_params = parameter_manager.get_parameters_from_json() + analysis_mode = workflow_params.get("analysis-mode", "LFQ") - # Get group mapping from parameters - param_manager = ParameterManager(workflow_dir) - params = param_manager.get_parameters_from_json() - group_map = { - key[11:]: value # Remove "mzML-group-" prefix - for key, value in params.items() - if key.startswith("mzML-group-") and value - } + if analysis_mode == "LFQ": + if not quant_dir.exists(): + return None - if not group_map: - return None + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None - df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) - df["Group"] = df["Reference"].map(group_map) - df = df.dropna(subset=["Group"]) + csv_file = csv_files[0] - groups = sorted(df["Group"].unique()) + try: + df = pd.read_csv(csv_file) + except Exception: + return None - if len(groups) < 2: - return None + if df.empty: + return None - group1, group2 = groups[:2] - - # Compute statistics per protein - stats_rows = [] - for protein, protein_df in df.groupby("ProteinName"): - g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values - g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values + # Get optional group mapping from parameters. + # Group information is not required at this stage; statistical testing + # happens in the Statistical page. + param_manager = ParameterManager(workflow_dir) + params = param_manager.get_parameters_from_json() + group_map = { + key[11:]: value # Remove "mzML-group-" prefix + for key, value in params.items() + if key.startswith("mzML-group-") and value + } - if len(g1_vals) < 2 or len(g2_vals) < 2: - pval = np.nan + df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) + + # Build sample display order. + if group_map: + sample_group_df = df[["Sample", "Reference"]].drop_duplicates() + sample_group_df["Group"] = sample_group_df["Reference"].map(group_map) + grouped_samples = [] + for grp in sorted(sample_group_df["Group"].dropna().unique()): + grouped_samples.extend( + sample_group_df[sample_group_df["Group"] == grp]["Sample"].tolist() + ) + remaining_samples = [ + s for s in sorted(df["Sample"].unique()) if s not in grouped_samples + ] + all_samples = grouped_samples + remaining_samples else: - _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False) - - mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan - mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan - - log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan + all_samples = sorted(df["Sample"].unique()) + + # Build pivot table + pivot_list = [] + for protein, group_df in df.groupby("ProteinName"): + peptides = ";".join(group_df["PeptideSequence"].unique()) + intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() + intensity_dict_complete = { + sample: intensity_dict.get(sample, 0) + for sample in all_samples + } + row = { + "ProteinName": protein, + **intensity_dict_complete, + "PeptideSequence": peptides, + } + pivot_list.append(row) + + pivot_df = pd.DataFrame(pivot_list) + pivot_df = pivot_df[["ProteinName"] + all_samples + ["PeptideSequence"]] + + # Build expression matrix (log2-transformed) + expr_df = pivot_df.set_index("ProteinName")[all_samples] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() + + return pivot_df, expr_df, group_map + + else: + if not quant_dir.exists(): + return None + + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None + + csv_file = csv_files[0] + + try: + df = pd.read_csv(csv_file, sep="\t", comment="#", engine="python") + except Exception: + return None + + if df.empty: + return None + + # ratio column removal + df = df.loc[:, ~df.columns.str.contains('ratio', case=False)] + + # exclude_indices = st.session_state.get("tmt_exclude_indices", []) + # group_map = st.session_state.get("tmt_group_map", {}) + # Get group mapping from parameters + parameter_manager = ParameterManager(Path(workflow_dir), "TOPP Workflow") + params = parameter_manager.get_parameters_from_json() + group_map = {} + for key, value in params.items(): + if key.startswith("TMT-group-") and value: + # Extract the numeric part from keys like "TMT-group-sample1" + match = re.search(r'sample(\d+)', key) + if match: + # Subtract 1 to convert to a 0-based index (0, 1, 2...). + # If your samples are already 0-based, remove the -1 adjustment. + index = str(int(match.group(1)) - 1) + group_map[index] = value + + # 1. Extract keys labeled as "skip" from group_map as integer list + exclude_indices = [ + int(k) for k, v in group_map.items() if v.lower() == "skip" + ] + + # 2. Remove "skip" entries from group_map (keep only actual group info) + group_map = { + int(k): v for k, v in group_map.items() if v.lower() != "skip" + } - stats_rows.append({ - "ProteinName": protein, - "log2FC": log2fc, - "p-value": pval, - }) + start_column_offset = 4 - stats_df = pd.DataFrame(stats_rows) + # st.write("exclude_indices:", exclude_indices) + # st.write("group_map:", group_map) - if not stats_df.empty: - mask = stats_df["p-value"].notna() - if mask.any(): - _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh") - stats_df.loc[mask, "p-adj"] = p_adj + if exclude_indices: + # st.write("Current columns:", df.columns.tolist()) + # st.write("Number of columns:", len(df.columns)) + # st.write("Exclude indices:", exclude_indices) + # st.write("Offset:", start_column_offset) + cols_to_drop = [df.columns[i + start_column_offset] for i in exclude_indices] + df_cleaned = df.drop(columns=cols_to_drop) else: - stats_df["p-adj"] = np.nan - - # Order samples by group (group2 first, then group1) - sample_group_df = df[["Sample", "Group"]].drop_duplicates() - group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist() - group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist() - all_samples = group2_samples + group1_samples - - # Build pivot table - pivot_list = [] - for protein, group_df in df.groupby("ProteinName"): - peptides = ";".join(group_df["PeptideSequence"].unique()) - intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() - intensity_dict_complete = { - sample: intensity_dict.get(sample, 0) - for sample in all_samples - } - row = { - "ProteinName": protein, - **intensity_dict_complete, - "PeptideSequence": peptides, - } - pivot_list.append(row) + df_cleaned = df.copy() + + current_cols = df_cleaned.columns.tolist() + sample_cols = current_cols[start_column_offset:] - pivot_df = pd.DataFrame(pivot_list) - pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left") - pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]] + # Ensure sample columns are numeric for downstream preprocessing/statistics. + pivot_df = df_cleaned.copy() + if sample_cols: + pivot_df[sample_cols] = pivot_df[sample_cols].apply(pd.to_numeric, errors='coerce') - # Build expression matrix (log2-transformed) - expr_df = pivot_df.set_index("ProteinName")[all_samples] - expr_df = expr_df.replace(0, np.nan) - expr_df = np.log2(expr_df + 1) - expr_df = expr_df.dropna() + protein_col = pivot_df.columns[0] + expr_df = pivot_df.set_index(protein_col)[sample_cols] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() - return pivot_df, expr_df, group_map + return pivot_df, expr_df, group_map def get_abundance_data(workspace: Path) -> tuple | None: - """Wrapper that handles cache key (workspace + CSV mtime). + """Wrapper that handles cache key (workspace + CSV mtime + params mtime). Args: workspace: Path to the workspace directory @@ -324,4 +376,49 @@ def get_abundance_data(workspace: Path) -> tuple | None: return None csv_mtime = csv_files[0].stat().st_mtime - return load_abundance_data(str(workspace), csv_mtime) + + params_file = workflow_dir / "params.json" + params_mtime = params_file.stat().st_mtime if params_file.exists() else 0.0 + + return load_abundance_data(str(workspace), csv_mtime, params_mtime) + + +def get_id_column(workspace: Path, pivot_df: pd.DataFrame) -> str: + """Resolve the protein/row identifier column for the active analysis mode. + + LFQ reports always use "ProteinName"; TMT reports use whatever the + report's first column is actually named (e.g. "protein"). + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + return "ProteinName" if analysis_mode == "LFQ" else pivot_df.columns[0] + + +def get_sample_group_map(workspace: Path, pivot_df: pd.DataFrame, group_map: dict) -> dict: + """Normalize group_map into {actual_sample_column_name: group_name}. + + LFQ group_map keys are already clean sample names (optionally with a + ".mzML" suffix). TMT group_map keys are 0-based channel indices that must + be matched against the report's actual "sampleN[...]" column names. + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + + if analysis_mode == "LFQ": + return { + k[:-5] if k.endswith(".mzML") else k: v + for k, v in group_map.items() + } + + actual_sample_names = pivot_df.columns.tolist() + norm_map = {} + for k, v in group_map.items(): + try: + sample_idx = int(k) + 1 + except (TypeError, ValueError): + continue + target_substring = f"sample{sample_idx}[" + real_full_name = next((name for name in actual_sample_names if target_substring in name), None) + if real_full_name: + norm_map[real_full_name] = v if v and v.strip() else "Unassigned" + return norm_map From e7dd66b9ec8666633de6dac4edccefc8722d4137 Mon Sep 17 00:00:00 2001 From: Yoo HoJun Date: Thu, 16 Jul 2026 15:15:14 +0900 Subject: [PATCH 07/10] Add LFQ/TMT workflow mode split and downstream analysis pages Split WorkflowTest configure() into separate LFQ and TMT tool tabs, and add new preprocessing/analysis pages (filtering, normalization, imputation, statistical testing, GO enrichment, clustered heatmap, pathway analysis) built on openms_insight engine functions. --- app.py | 20 +- content/enrichment.py | 141 ++ content/filtering.py | 173 +++ content/imputation.py | 145 ++ content/normalization.py | 242 ++++ content/results_abundance.py | 138 +- content/results_heatmap.py | 79 +- content/results_heatmap_clustered.py | 107 ++ content/results_pathway_analysis.py | 258 ++++ content/results_pca.py | 177 ++- content/results_proteomicslfq.py | 5 +- content/results_volcano.py | 91 +- content/statistical.py | 165 +++ requirements.txt | 8 +- src/WorkflowTest.py | 1841 +++++++++++++++++--------- src/common/results_helpers.py | 287 ++-- 16 files changed, 2995 insertions(+), 882 deletions(-) create mode 100644 content/enrichment.py create mode 100644 content/filtering.py create mode 100644 content/imputation.py create mode 100644 content/normalization.py create mode 100644 content/results_heatmap_clustered.py create mode 100644 content/results_pathway_analysis.py create mode 100644 content/statistical.py diff --git a/app.py b/app.py index 194d857..e76dcd8 100644 --- a/app.py +++ b/app.py @@ -1,3 +1,10 @@ +import os +# Polars' default (CPU-core-count-sized) native thread pool access-violation +# crashes the whole process on some high-core-count Windows machines when +# invoked from Streamlit's script-runner thread. Must be set before polars +# is imported anywhere (including transitively via openms_insight). +os.environ.setdefault("POLARS_MAX_THREADS", "1") + import streamlit as st from pathlib import Path import json @@ -23,12 +30,19 @@ st.Page(Path("content", "results_rescoring.py"), title="Rescoring", icon="πŸ“ˆ"), st.Page(Path("content", "results_filtered.py"), title="Filtered PSMs", icon="🎯"), st.Page(Path("content", "results_abundance.py"), title="Abundance", icon="πŸ“‹"), + + ], + "Differential Protein Analysis": [ + st.Page(Path("content", "filtering.py"), title="Filtering", icon="🧹"), + st.Page(Path("content", "imputation.py"), title="Imputation", icon="🩹"), + st.Page(Path("content", "normalization.py"), title="Normalization", icon="βš–οΈ"), + st.Page(Path("content", "statistical.py"), title="Statistical", icon="πŸ”’"), st.Page(Path("content", "results_volcano.py"), title="Volcano", icon="πŸŒ‹"), st.Page(Path("content", "results_pca.py"), title="PCA", icon="πŸ“Š"), st.Page(Path("content", "results_heatmap.py"), title="Heatmap", icon="πŸ”₯"), - st.Page(Path("content", "results_library.py"), title="Spectral Library", icon="πŸ“š"), - st.Page(Path("content", "results_proteomicslfq.py"), title="Proteomics LFQ", icon="πŸ§ͺ"), - ], + st.Page(Path("content", "results_heatmap_clustered.py"), title="Clustered Heatmap", icon="🧬"), + st.Page(Path("content", "enrichment.py"), title="Pathway Analysis", icon="πŸ“‰"), + ] } pg = st.navigation(pages) diff --git a/content/enrichment.py b/content/enrichment.py new file mode 100644 index 0000000..62fc9fa --- /dev/null +++ b/content/enrichment.py @@ -0,0 +1,141 @@ +"""Pathway Analysis Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column +# Import GO Enrichment modules from openms_insight engine +from openms_insight.analysis.enrichment import calculate_go_enrichment + +params = page_setup() +st.title("GO Enrichment Analysis") + +st.markdown( + """ +Identify overrepresented biological themes (BP, CC, MF) within your differentially expressed protein features using MyGene.info and Fisher's Exact Test. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# --- STEP 1: Upstream Statistics Checkpoint --- +if ( + "statistics_df" in st.session_state + and st.session_state["statistics_df"] is not None +): + final_statistics_report = st.session_state["statistics_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using analyzed matrices from the **Statistical Inference** step." + ) +else: + st.warning( + "⚠️ **Missing Prerequisites**: Statistical inference data not detected. Please run hypothesis testing first." + ) + st.page_link( + "content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬" + ) + st.stop() + +# --- STEP 2: Preprocessing Mapping Key Configuration --- +# Identify target identifier columns dynamically +abundance_result = get_abundance_data(st.session_state["workspace"]) +id_col = get_id_column(st.session_state["workspace"], abundance_result[0]) if abundance_result else "ProteinName" +if id_col not in final_statistics_report.columns: + st.error(f"❌ Structural Error: Column '{id_col}' is missing from the active matrix context.") + st.stop() + +# --- SECTION 1: Parameter Setup & Dynamic Cutoff Labels --- +st.subheader("Configure Enrichment Thresholds") + +# Check if target p-value should be adjusted or raw based on previous selections (Fallback safely to 'p-adj') +target_p_col = "p-adj" if "p-adj" in final_statistics_report.columns else "p-value" +p_label = ( + "Adjusted P-value (p-adj) Cutoff" + if target_p_col == "p-adj" + else "Raw P-value (p-value) Cutoff" +) + +ui_go_col1, ui_go_col2 = st.columns(2) + +with ui_go_col1: + p_cutoff = st.number_input( + f"πŸ”¬ {p_label}", + min_value=0.0001, + max_value=1.0, + value=0.05, + step=0.01, + format="%.4f", + help="Proteins with significance metrics below this value are mapped to the foreground cohort.", + ) + +with ui_go_col2: + fc_cutoff = st.number_input( + "πŸ“ˆ Absolute Difference Cutoff (|log2FC|)", + min_value=0.0, + max_value=10.0, + value=1.0, + step=0.1, + format="%.2f", + help="Proteins with absolute log2 fold change greater than or equal to this threshold will be selected.", + ) + +# --- SECTION 2: Execution and Interactive View Charts --- +st.markdown("
", unsafe_allow_html=True) +if st.button("πŸš€ Run GO Enrichment Analysis", type="primary", key="run_go_analysis"): + + with st.spinner("Querying MyGene.info API & executing hyper-geometric calculation loops..."): + # Convert internal pandas DataFrame to openms_insight Polars DataFrame expectation + stats_pl = pl.from_pandas(final_statistics_report) + + status, output = calculate_go_enrichment( + final_report=stats_pl, + id_col=id_col, + target_p_col=target_p_col, + p_cutoff=p_cutoff, + fc_cutoff=fc_cutoff, + ) + + # Route response structures based on analysis output status code + if status == "empty_data": + st.error("❌ No valid statistical rows found containing standard columns to run GO alignment.") + + elif status == "insufficient_proteins": + st.warning( + f"⚠️ Not enough significant proteins found to construct target datasets. " + f"(Criteria: {target_p_col} < {p_cutoff:.4f}, |log2FC| β‰₯ {fc_cutoff:.2f})." + ) + st.info(f"πŸ’‘ Found significant proteins count: **{output}**. Try relaxing your p-value or log2FC filters.") + + elif status == "success": + st.success("β­• GO Enrichment Analysis completed successfully!") + + # Display operational matrix scale + st.markdown( + f"πŸ“Š **Analysis Profile Scope**: Mapped **{output['fg_count']}** significant foreground profiles out of **{output['bg_count']}** reference background items." + ) + + # Build multi-tab interface layer for ontology subcategories + tabs = st.tabs([ + "🧬 Biological Process (BP)", + "πŸ”¬ Cellular Component (CC)", + "πŸ§ͺ Molecular Function (MF)" + ]) + categories_data = output["categories"] + + for idx, go_type in enumerate(["BP", "CC", "MF"]): + with tabs[idx]: + fig = categories_data[go_type]["fig"] + df_go = categories_data[go_type]["df"] + + if fig is not None and df_go is not None: + # Render plotly bar figures generated straight from backend engine + st.plotly_chart(fig, use_container_width=True) + + st.subheader(f"πŸ“Š {go_type} Results Dataframe") + st.dataframe(df_go, use_container_width=True) + else: + st.info(f"No statistically overrepresented terms identified for Category: **{go_type}**") \ No newline at end of file diff --git a/content/filtering.py b/content/filtering.py new file mode 100644 index 0000000..2bad00c --- /dev/null +++ b/content/filtering.py @@ -0,0 +1,173 @@ +"""Filtering Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import filtering functions from openms_insight package +from openms_insight.analysis.filter import ( + filter_low_abundance, + filter_low_repeatability, + filter_low_variance, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Filtering") + +st.markdown( + """ +Filter out low-quality proteins from your dataset based on abundance, repeatability, or variance thresholds. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Identify actual sample columns dynamically +sample_cols = [ + c + for c in pivot_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Original Data View --- +st.subheader("Original Abundance Table") +st.markdown( + f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering." +) +st.dataframe(pivot_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Filter Configuration --- +st.subheader("Configure Filter Engine") + +# Prepare Polars Metadata DataFrame required by openms_insight functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for filtering strategy +filter_method = st.selectbox( + "Select Filtering Method", + options=["Low Abundance", "Low Repeatability", "Low Variance"], + index=0, + help="Choose the statistical criteria to prune unreliable protein entries.", +) + +# Render threshold sliders dynamically based on the selected filter method +if filter_method == "Low Abundance": + st.markdown( + "**Low Abundance Filter**: Keeps rows where at least one group's median is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +elif filter_method == "Low Repeatability": + st.markdown( + "**Low Repeatability Filter**: Keeps rows where at least one group has a missing value ratio within the allowed maximum." + ) + threshold = st.slider( + "Max Missing Ratio", + min_value=0.0, + max_value=100.0, + value=50.0, + step=5.0, + help="Allowed missing value (zero or null) ratio per group.", + ) + +elif filter_method == "Low Variance": + st.markdown( + "**Low Variance Filter**: Keeps rows where at least one group's variance is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +# --- SECTION 3: Filter Execution and Collected Results View --- +if st.button("Apply Filter", type="primary"): + # Convert the original Pandas DataFrame into a Polars LazyFrame graph + quant_lazy = pl.from_pandas(pivot_df).lazy() + + # Route execution to the chosen openms_insight engine function + if filter_method == "Low Abundance": + filtered_lazy = filter_low_abundance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + elif filter_method == "Low Repeatability": + # Convert percent slider input to ratio expected by the function (e.g., 50.0% -> 0.5) + filtered_lazy = filter_low_repeatability( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + max_missing_ratio=threshold / 100.0, + ) + elif filter_method == "Low Variance": + filtered_lazy = filter_low_variance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + + # Collect the evaluated lazy graph and convert back to Pandas for visualization + filtered_df = strip_stat_columns(filtered_lazy.collect().to_pandas()) + st.session_state["filtered_df"] = filtered_df + + # Layout response metrics and the filtered matrix + st.success(f"Successfully applied **{filter_method}** filter!") + + # Display dataset scale compression stats + col1, col2, col3 = st.columns(3) + col1.metric("Original Proteins", pivot_df.shape[0]) + col2.metric("Filtered Proteins", filtered_df.shape[0]) + col3.metric( + "Removed Proteins", pivot_df.shape[0] - filtered_df.shape[0], delta=None + ) + + st.subheader("Filtered Abundance Table") + if filtered_df.empty: + st.warning( + "The filtered table is empty. Try relaxing the threshold constraints." + ) + else: + st.dataframe(filtered_df, use_container_width=True) \ No newline at end of file diff --git a/content/imputation.py b/content/imputation.py new file mode 100644 index 0000000..4350263 --- /dev/null +++ b/content/imputation.py @@ -0,0 +1,145 @@ +"""Imputation Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import imputation algorithms from openms_insight engine +from openms_insight.analysis.imputation import impute_mar, impute_smallest_value + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Missing Value Imputation") + +st.markdown( + """ +Handle missing values (zeros or nulls) in your quantification matrix using biological group-aware (MAR) or absolute lowest limit (MNAR) techniques. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load base dataset and clean dictionary keys +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Pipeline Checkpoint: Fetch upstream filtered data if available, fallback to raw pivot matrix +if "filtered_df" in st.session_state and st.session_state["filtered_df"] is not None: + base_df = strip_stat_columns(st.session_state["filtered_df"]) + st.session_state["filtered_df"] = base_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No filtering history found. Operating on the original unfiltered table." + ) + +# 2. Identify actual sample columns dynamically based on the current active matrix +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Input Matrix Summary --- +st.subheader("Input Matrix Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Imputation Configuration --- +st.subheader("Configure Imputation Engine") + +# Build Polars structural metadata DataFrame +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for core missingness assumption strategy +impute_category = st.selectbox( + "Select Imputation Class", + options=["MAR (Missing At Random)", "MNAR (Missing Not At Random)"], + index=0, + help="MAR uses group metrics (Mean/Median). MNAR shifts values below the limit of detection.", +) + +# Render algorithmic options sub-menus based on the parent selection +if impute_category == "MAR (Missing At Random)": + st.markdown( + "**Group Character Imputation**: Fills missing metrics leveraging sample properties belonging to the same group." + ) + strategy_opt = st.radio( + "Mathematical Strategy", + options=["median", "mean"], + index=0, + horizontal=True, + ) + +elif impute_category == "MNAR (Missing Not At Random)": + st.markdown( + "**Smallest Value Imputation**: Replaces missing items with the minimum values detected to reflect technical dropout limits." + ) + scope_opt = st.radio( + "Detection Minimum Scope", + options=["row", "global"], + index=0, + horizontal=True, + help="'row' targets current protein minimum; 'global' searches the entire mass spectrometry matrix profile.", + ) + +# --- SECTION 3: Imputation Execution --- +if st.button("Apply Imputation", type="primary"): + # Initialize optimization pipeline graph via lazy loading conversion + quant_lazy = pl.from_pandas(base_df).lazy() + + # Route configuration matrix parameters to designated engine function channels + if impute_category == "MAR (Missing At Random)": + imputed_lazy = impute_mar( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + strategy=strategy_opt, + ) + elif impute_category == "MNAR (Missing Not At Random)": + imputed_lazy = impute_smallest_value( + quantification_data=quant_lazy, metadata=metadata_pl, scope=scope_opt + ) + + # Resolve lazy graph optimization tree and push to display data frame structure + imputed_df = strip_stat_columns(imputed_lazy.collect().to_pandas()) + + # πŸ’Ύ Save current output into Session State for down-stream processing (Normalization, Statistics) + st.session_state["imputed_df"] = imputed_df + + st.success(f"Successfully finalized **{impute_category}** imputation step!") + + # Calculate and display a quick performance matrix check + st.subheader("Imputed Result Table") + st.dataframe(imputed_df, use_container_width=True) \ No newline at end of file diff --git a/content/normalization.py b/content/normalization.py new file mode 100644 index 0000000..c0e97e8 --- /dev/null +++ b/content/normalization.py @@ -0,0 +1,242 @@ +"""Normalization Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import normalization engine functions from openms_insight +from openms_insight.analysis.normalization import ( + normalize_samples, + scale_data, + transform_data, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None: + """Keep preprocessing tables intensity-only before statistical analysis.""" + if df is None: + return None + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Normalization & Scaling") + +st.markdown( + """ +Standardize and transform your protein abundance profiles to correct for technical variations and optimize statistical distributions. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +filtered_df = strip_stat_columns(st.session_state.get("filtered_df")) +imputed_df = strip_stat_columns(st.session_state.get("imputed_df")) +normalized_df = strip_stat_columns(st.session_state.get("normalized_df")) +if filtered_df is not None: + st.session_state["filtered_df"] = filtered_df +if imputed_df is not None: + st.session_state["imputed_df"] = imputed_df +if normalized_df is not None: + st.session_state["normalized_df"] = normalized_df + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = imputed_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = filtered_df + st.warning( + "⚠️ **Imputation Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original unfiltered table." + ) + +# 2. Extract actual active sample columns dynamically +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("### Pipeline Overview") +st.caption("Data flows in order: Filtering -> Imputation -> Normalization") + +step_rows = [ + { + "Step": "Filtering", + "Status": "Done" if filtered_df is not None else "Not run", + "Rows": filtered_df.shape[0] if filtered_df is not None else "-", + "Cols": filtered_df.shape[1] if filtered_df is not None else "-", + }, + { + "Step": "Imputation", + "Status": "Done" if imputed_df is not None else "Not run", + "Rows": imputed_df.shape[0] if imputed_df is not None else "-", + "Cols": imputed_df.shape[1] if imputed_df is not None else "-", + }, + { + "Step": "Normalization", + "Status": "Done" if normalized_df is not None else "Not run", + "Rows": normalized_df.shape[0] if normalized_df is not None else "-", + "Cols": normalized_df.shape[1] if normalized_df is not None else "-", + }, +] +st.dataframe(pd.DataFrame(step_rows), hide_index=True, use_container_width=True) + +with st.expander("Show step tables", expanded=False): + if filtered_df is not None: + st.markdown("#### Filtering output") + st.dataframe(filtered_df.head(10), use_container_width=True) + if imputed_df is not None: + st.markdown("#### Imputation output") + st.dataframe(imputed_df.head(10), use_container_width=True) + if normalized_df is not None: + st.markdown("#### Normalization output") + st.dataframe(normalized_df.head(10), use_container_width=True) + if filtered_df is None and imputed_df is None and normalized_df is None: + st.info("No preprocessing outputs yet. Start from Filtering.") + +st.markdown("---") + +# --- SECTION 2: Normalization Parameter Configuration --- +st.subheader("Configure Preprocessing & Scaling Chains") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2, col3 = st.columns(3) + +with col1: + st.markdown("### 🧬 1. Mathematical Transformation") + transform_strategy = st.selectbox( + "Select Transformation", + options=["None", "log2", "log10", "square_root", "cube_root"], + index=0, + help="Compress data dynamic range and stabilize heteroscedastic variance profiles.", + ) + +with col2: + st.markdown("### πŸ§ͺ 2. Sample Normalization") + norm_strategy = st.selectbox( + "Select Normalization", + options=["None", "sum", "median", "pqn", "reference_feature", "quantile"], + index=0, + help="Perform column-wise corrections to account for variable sample loading concentrations.", + ) + + # Conditionally display target input field for reference feature matching + ref_feature_input = None + if norm_strategy == "reference_feature": + ref_feature_input = st.text_input( + "Reference Protein Name (ID)", + value="", + placeholder="e.g., P01234 or GAPDH", + help=f"Enter the exact unique identifier string matching a key inside the '{id_col}' column.", + ) + +with col3: + st.markdown("### πŸ“Š 3. Row Scaling") + scaling_strategy = st.selectbox( + "Select Scaling Mode", + options=["None", "mean_centering", "auto_scaling", "pareto_scaling", "range_scaling"], + index=0, + help="Adjust individual feature weights to make low and high abundance proteins comparable.", + ) + + +# --- SECTION 3: Normalization Pipe Sequential Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Apply Normalization Pipelines", type="primary"): + + # Validate reference feature selection if active before hitting polars execution layers + if norm_strategy == "reference_feature" and not ref_feature_input: + st.error( + "❌ Validation Error: Please provide a valid Reference Protein Name to use the 'reference_feature' strategy." + ) + st.stop() + + # Convert pandas memory buffer into optimization lazy dataframe tree graph + processing_lazy = pl.from_pandas(base_df).lazy() + + # Execute Chain 1: Transform Matrix Data + try: + processing_lazy = transform_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=transform_strategy, + ) + + # Execute Chain 2: Normalize Sample Intensities (Columns) + processing_lazy = normalize_samples( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=norm_strategy, + id_col=id_col, + reference_feature=ref_feature_input if norm_strategy == "reference_feature" else None, + ) + + # Execute Chain 3: Scale Individual Features (Rows) + processing_lazy = scale_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=scaling_strategy, + ) + + # Finalize and collect pipeline query graph optimizations + normalized_df = strip_stat_columns(processing_lazy.collect().to_pandas()) + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (Statistics Block) + st.session_state["normalized_df"] = normalized_df + + st.success("Successfully executed all selected normalization pipelines!") + + # Display the finalized transformation matrix view + st.subheader("Normalized Abundance Table") + st.dataframe(normalized_df, use_container_width=True) + + except ValueError as val_err: + # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein) + st.error(f"Engine Configuration Error: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/content/results_abundance.py b/content/results_abundance.py index a7ff453..38c42bc 100644 --- a/content/results_abundance.py +++ b/content/results_abundance.py @@ -1,9 +1,11 @@ """Abundance (ProteomicsLFQ) Results Page.""" import streamlit as st import pandas as pd +import numpy as np from pathlib import Path from src.common.common import page_setup from src.common.results_helpers import get_workflow_dir, get_abundance_data +from src.workflow.ParameterManager import ParameterManager params = page_setup() st.title("Abundance Quantification") @@ -11,7 +13,7 @@ st.markdown( """ View protein and PSM-level quantification from **ProteomicsLFQ**. -This page calculates differential expression statistics between sample groups. +This page focuses on raw abundance intensity for preprocessing. """ ) @@ -21,6 +23,10 @@ workflow_dir = get_workflow_dir(st.session_state["workspace"]) quant_dir = workflow_dir / "results" / "quant_results" +parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") + +workflow_params = parameter_manager.get_parameters_from_json() +analysis_mode = workflow_params.get("analysis-mode", "LFQ") if not quant_dir.exists(): st.info("No quantification results available yet. Please run the workflow first.") @@ -35,6 +41,55 @@ csv_file = csv_files[0] +def render_protein_table(pivot_df, is_lfq=True): + """Common function to render the protein-level abundance table""" + pivot_df = pivot_df.copy() + st.markdown("### Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples." + ) + + if is_lfq: + # Handle LFQ mode columns (Raw Intensity) + id_col = "ProteinName" + exclude_cols = [id_col, "PeptideSequence"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + display_cols = [id_col, "Intensity"] + sample_cols + ["PeptideSequence"] + help_text = "Raw sample intensities" + y_min = None + else: + # Handle non-LFQ mode columns (Log2-transformed Intensity) + id_col = "protein" + exclude_cols = [id_col, "n_proteins", "n_peptides", "protein_score"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols and "ratio" not in c.lower()] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply( + lambda row: [np.log2(v + 1) for v in row], axis=1 + ) + display_cols = [id_col, "Intensity"] + sample_cols + help_text = "Sample intensities (log2 scale)" + y_min = 0 + + # Filter to available columns, then sort and display + available_cols = [c for c in display_cols if c in pivot_df.columns] + view_df = pivot_df[available_cols] + + st.dataframe( + view_df, + column_config={ + "Intensity": st.column_config.BarChartColumn( + "Intensity", + help=help_text, + width="small", + y_min=y_min, + ), + }, + use_container_width=True, + ) + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) try: @@ -44,68 +99,53 @@ st.info("No data found in this file.") st.stop() - with protein_tab: - st.markdown("### Protein-Level Abundance Table") + result = get_abundance_data(st.session_state["workspace"]) - st.info( - "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." - ) + if analysis_mode == "LFQ": + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) - result = get_abundance_data(st.session_state["workspace"]) - if result is None: - st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.") - st.page_link("content/workflow_configure.py", label="Go to Configure", icon="βš™οΈ") - st.stop() + with protein_tab: + if result is None: + st.warning("Could not load abundance data. Please run the workflow first.") + st.stop() + + pivot_df, expr_df, group_map = result + render_protein_table(pivot_df, is_lfq=True) - pivot_df, expr_df, group_map = result + with psm_tab: + st.markdown("### PSM-level Quantification Table") + st.info( + "This table shows the PSM-level quantification data, including protein IDs, " + "peptide sequences, charge states, and intensities across samples. " + "Each row represents one peptide-spectrum match detected from the MS/MS analysis." + ) + st.dataframe(df, use_container_width=True) - # Display group comparison info - groups = sorted(set(group_map.values())) - if len(groups) >= 2: - group1, group2 = sorted(groups)[:2] - st.info(f"Statistical comparison: **{group2} vs {group1}**") + else: + pre_processing_tab, protein_tab = st.tabs(["Pre-processing", "Protein Table"]) - # Get sample columns (between stats and PeptideSequence) - sample_cols = [c for c in pivot_df.columns if c not in ["ProteinName", "log2FC", "p-value", "PeptideSequence"]] + if result is None: + st.info("πŸ’‘ Please run the workflow first to see results.") + st.stop() - pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + pivot_df, expr_df, group_map = result - # Reorder columns: place Intensity after p-value - display_cols = ["ProteinName", "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"] - display_df = pivot_df[display_cols] - - st.dataframe( - display_df.sort_values("p-value"), - column_config={ - "Intensity": st.column_config.BarChartColumn( - "Intensity", - help="Raw sample intensities", - width="small", - ), - }, - use_container_width=True, - ) + with pre_processing_tab: + st.write("### Final Results (Intensity matrix)") + st.dataframe(pivot_df.head(10)) - with psm_tab: - st.markdown("### PSM-level Quantification Table") - st.info( - "This table shows the PSM-level quantification data, including protein IDs, " - "peptide sequences, charge states, and intensities across samples. " - "Each row represents one peptide-spectrum match detected from the MS/MS analysis." - ) - st.dataframe(df, use_container_width=True) + with protein_tab: + render_protein_table(pivot_df, is_lfq=False) except Exception as e: st.error(f"Failed to load {csv_file.name}: {e}") st.markdown("---") -st.markdown("**Next steps:** Explore statistical visualizations") +st.markdown("**Next steps:** Continue preprocessing, then run statistical inference") col1, col2, col3 = st.columns(3) with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") + st.page_link("content/filtering.py", label="Filtering", icon="🧹") with col2: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") + st.page_link("content/imputation.py", label="Imputation", icon="🧩") with col3: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") + st.page_link("content/statistical.py", label="Statistical Inference", icon="πŸ”¬") \ No newline at end of file diff --git a/content/results_heatmap.py b/content/results_heatmap.py index 4ece3f4..104bff6 100644 --- a/content/results_heatmap.py +++ b/content/results_heatmap.py @@ -1,19 +1,18 @@ """Heatmap Results Page.""" import streamlit as st import numpy as np -import plotly.express as px -from scipy.cluster.hierarchy import linkage, leaves_list -from scipy.spatial.distance import pdist +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import Heatmap params = page_setup() st.title("Heatmap") st.markdown( """ -Hierarchically clustered heatmap of protein-level abundance (Z-score normalized). -Proteins and samples are ordered by similarity. +Interactive hierarchically clustered heatmap of protein-level abundance (Z-score normalized). +Powered by OpenMS-Insight multi-resolution engine. """ ) @@ -28,42 +27,68 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) -top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +sample_cols = expr_df.columns.tolist() +# UI settings (number of top variance proteins) +top_n = st.slider("Number of proteins (Highest Variance)", 20, 200, 50, key="heatmap_top_n") + +# Process data (variance selection -> Z-score normalization) var_series = expr_df.var(axis=1) top_proteins = var_series.sort_values(ascending=False).head(top_n).index heatmap_df = expr_df.loc[top_proteins] + +# Compute Z-scores and clean missing/invalid values heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) + # Melt and convert data to Polars to satisfy OpenMS-Insight component requirements + # Restore the id column from the index as a regular column + heatmap_z_reset = heatmap_z.reset_index() - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) + # Unpivot the wide-format matrix into long-format (X, Y, Intensity) + melted_df = heatmap_z_reset.melt( + id_vars=[id_col], + value_vars=sample_cols, + var_name="Sample", + value_name="Z_score" + ) - heatmap_clustered = heatmap_z.iloc[row_order, col_order] + # Add sample group mapping if available for heatmap categories + if sample_group_map: + melted_df["Group"] = melted_df["Sample"].map(sample_group_map) - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) + # Pack the Pandas DataFrame into a Polars LazyFrame + heatmap_pl_lazy = pl.from_pandas(melted_df).lazy() - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} + # Initialize the OpenMS-Insight Heatmap component and map attributes + heatmap_component = Heatmap( + cache_id="quantms_protein_heatmap", + x_column="Sample", + y_column=id_col, + data=heatmap_pl_lazy, + intensity_column="Z_score", + title="Protein Abundance Heatmap (Z-score)", + x_label="Samples", + y_label="Proteins", + colorscale="RdBu", + reversescale=True, + log_scale=False, # Z-score can be negative, so log scale must stay off + intensity_label="Z-score", + category_column=None, + min_points=10000, # Generous point-count ceiling so the full grid renders ) - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, use_container_width=True) + # Render the component + state_manager = st.session_state.get("state") + heatmap_component(state_manager=state_manager) else: st.warning("Insufficient data to generate the heatmap.") diff --git a/content/results_heatmap_clustered.py b/content/results_heatmap_clustered.py new file mode 100644 index 0000000..7104c3a --- /dev/null +++ b/content/results_heatmap_clustered.py @@ -0,0 +1,107 @@ +"""Clustered Heatmap Results Page.""" +import streamlit as st +import numpy as np +import polars as pl +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import ClusteredHeatmap + +params = page_setup() +st.title("Clustered Heatmap") + +st.markdown( + """ +A real grid heatmap (rows = proteins, columns = samples) with hierarchical +clustering dendrograms on both axes and a sample-group color bar, powered +by OpenMS-Insight. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +top_n = st.slider("Number of proteins (Highest Variance)", 10, 200, 30, key="clustered_heatmap_top_n") + +var_series = expr_df.var(axis=1) +top_proteins = var_series.sort_values(ascending=False).head(top_n).index +heatmap_df = expr_df.loc[top_proteins] + +heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) +heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() + +if heatmap_z.empty: + st.warning("Insufficient data to generate the heatmap.") + st.stop() + +heatmap_z_reset = heatmap_z.reset_index() +heatmap_lazy = pl.from_pandas(heatmap_z_reset).lazy() + +sample_cols = expr_df.columns.tolist() +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, +) + +# Assign group annotation-bar colors in sorted-group order (matching how +# ClusteredHeatmap._preprocess() orders unique groups internally). +group_palette = [ + "#00BFC4", # teal + "#F8766D", # salmon + "#7CAE00", # yellow-green + "#C77CFF", # lavender purple + "#E7B800", # gold/amber + "#619CFF", # blue + "#FF61C3", # pink/magenta + "#00BA38", # green + "#FF8C42", # orange + "#00B0F6", # sky blue +] +unique_groups = sorted(set(sample_group_map.values())) +group_colors = {g: group_palette[i % len(group_palette)] for i, g in enumerate(unique_groups)} + +heatmap_component = ClusteredHeatmap( + cache_id="quantms_clustered_heatmap", + cache_path=str(st.session_state["workspace"]), + id_col=id_col, + data=heatmap_lazy, + metadata=metadata_pl, + row_cluster=True, + col_cluster=True, + title="Protein Abundance Heatmap (Z-score, clustered)", + x_label="Samples", + y_label="Proteins", + colorscale=[[0, "#6699E0"], [0.5, "#FFFFFF"], [1, "#E06666"]], + reversescale=False, + intensity_label="Z-score", + group_colors=group_colors, +) + +state_manager = st.session_state.get("state") +# Scale height with the number of proteins so row labels stay readable - +# BaseComponent otherwise defaults to a flat 400px, too short for a +# dendrogram+heatmap composite with more than a handful of rows. +heatmap_height = max(600, min(1400, 300 + top_n * 20)) +heatmap_component(state_manager=state_manager, height=heatmap_height) + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") +with col2: + st.page_link("content/results_heatmap.py", label="Heatmap (original)", icon="πŸ”₯") diff --git a/content/results_pathway_analysis.py b/content/results_pathway_analysis.py new file mode 100644 index 0000000..f5eb4c1 --- /dev/null +++ b/content/results_pathway_analysis.py @@ -0,0 +1,258 @@ +import json +import mygene +import streamlit as st +import pandas as pd +import numpy as np +import plotly.express as px +import plotly.io as pio +from collections import defaultdict +from scipy.stats import fisher_exact +from pathlib import Path +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data + +# ================================ +# Page setup +# ================================ +params = page_setup() +st.title("ProteomicsLFQ Results") + +# ================================ +# Workspace check +# ================================ +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# ================================ +# _run_go_enrichment function +# ================================ +def _run_go_enrichment(pivot_df: pd.DataFrame, results_dir: Path): + p_cutoff = 0.05 + fc_cutoff = 1.0 + + analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() + + if analysis_df.empty: + st.error("No valid statistical data found for GO enrichment.") + st.write("❗ analysis_df is empty") + else: + with st.spinner("Fetching GO terms from MyGene.info API..."): + mg = mygene.MyGeneInfo() + + def get_clean_uniprot(name): + parts = str(name).split("|") + return parts[1] if len(parts) >= 2 else parts[0] + + analysis_df["UniProt"] = analysis_df["protein"].apply(get_clean_uniprot) + + bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() + fg_ids = analysis_df[ + (analysis_df["p-value"] < p_cutoff) & + (analysis_df["log2FC"].abs() >= fc_cutoff) + ]["UniProt"].dropna().astype(str).unique().tolist() + # st.write("βœ… get_clean_uniprot applied") + + if len(fg_ids) < 3: + st.warning( + f"Not enough significant proteins " + f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " + f"Found: {len(fg_ids)}" + ) + st.write("❗ Not enough significant proteins") + else: + res_list = mg.querymany( + bg_ids, scopes="uniprot", fields="go", as_dataframe=False + ) + res_go = pd.DataFrame(res_list) + if "notfound" in res_go.columns: + res_go = res_go[res_go["notfound"] != True] + + def extract_go_terms(go_data, go_type): + if not isinstance(go_data, dict) or go_type not in go_data: + return [] + terms = go_data[go_type] + if isinstance(terms, dict): + terms = [terms] + return list({t.get("term") for t in terms if "term" in t}) + + for go_type in ["BP", "CC", "MF"]: + res_go[f"{go_type}_terms"] = res_go["go"].apply( + lambda x: extract_go_terms(x, go_type) + ) + + annotated_ids = set(res_go["query"].astype(str)) + fg_set = annotated_ids.intersection(fg_ids) + bg_set = annotated_ids + # st.write(f"βœ… fg_set bg_set are set") + + def run_go(go_type): + go2fg = defaultdict(set) + go2bg = defaultdict(set) + + for _, row in res_go.iterrows(): + uid = str(row["query"]) + for term in row[f"{go_type}_terms"]: + go2bg[term].add(uid) + if uid in fg_set: + go2fg[term].add(uid) + + records = [] + N_fg = len(fg_set) + N_bg = len(bg_set) + + for term, fg_genes in go2fg.items(): + a = len(fg_genes) + if a == 0: + continue + b = N_fg - a + c = len(go2bg[term]) - a + d = N_bg - (a + b + c) + + _, p = fisher_exact([[a, b], [c, d]], alternative="greater") + records.append({ + "GO_Term": term, + "Count": a, + "GeneRatio": f"{a}/{N_fg}", + "p_value": p, + }) + + df = pd.DataFrame(records) + if df.empty: + return None, None + + df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) + df = df.sort_values("p_value").head(20) + + # βœ… Plotly Figure + fig = px.bar( + df, + x="-log10(p)", + y="GO_Term", + orientation="h", + title=f"GO Enrichment ({go_type})", + ) + + # st.write(f"βœ… Plotly Figure generated") + + fig.update_layout( + yaxis=dict(autorange="reversed"), + height=500, + margin=dict(l=10, r=10, t=40, b=10), + ) + + return fig, df + + go_results = {} + + for go_type in ["BP", "CC", "MF"]: + fig, df_go = run_go(go_type) + if fig is not None: + go_results[go_type] = { + "fig": fig, + "df": df_go + } + # st.write(f"βœ… go_type generated") + + go_dir = results_dir / "go-terms" + go_dir.mkdir(parents=True, exist_ok=True) + + go_data = {} + + for go_type in ["BP", "CC", "MF"]: + if go_type in go_results: + fig = go_results[go_type]["fig"] + df = go_results[go_type]["df"] + + go_data[go_type] = { + "fig_json": fig.to_json(), # Figure β†’ JSON string + "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts + } + + go_json_file = go_dir / "go_results.json" + with open(go_json_file, "w") as f: + json.dump(go_data, f) + st.session_state["go_results"] = go_results + st.session_state["go_ready"] = True if go_data else False + # st.write("βœ… GO enrichment analysis complete") + +# ================================ +# Load abundance data +# ================================ +results_dir = Path(st.session_state["workspace"]) / "topp-workflow" / "results" / "quant_results" +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result + +go_json_file = results_dir / "go-terms" / "go_results.json" + +go_input_df = pivot_df.copy() +if "ProteinName" in go_input_df.columns: + go_input_df = go_input_df.rename(columns={"ProteinName": "protein"}) + +_run_go_enrichment(go_input_df, results_dir) + +# ================================ +# Tabs +# ================================ +protein_tab, = st.tabs(["🧬 Protein Table"]) + +# ================================ +# Protein-level results +# ================================ +with protein_tab: + st.markdown("### 🧬 Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples.\n\n" + "Additionally, log2 fold change and p-values are calculated between sample groups." + ) + + if pivot_df.empty: + st.info("No protein-level data available.") + else: + st.session_state["pivot_df"] = pivot_df + st.dataframe(pivot_df.sort_values("p-value"), width="stretch") + +# ====================================================== +# GO Enrichment Results +# ====================================================== +st.markdown("---") +st.subheader("🧬 GO Enrichment Analysis") + +if not go_json_file.exists(): + st.info("GO Enrichment results are not available yet. Please run the analysis first.") +else: + with open(go_json_file, "r") as f: + go_data = json.load(f) + + bp_tab, cc_tab, mf_tab = st.tabs([ + "🧬 Biological Process", + "🏠 Cellular Component", + "βš™οΈ Molecular Function", + ]) + + for tab, go_type in zip([bp_tab, cc_tab, mf_tab], ["BP", "CC", "MF"]): + with tab: + if go_type not in go_data: + st.info(f"No enriched {go_type} terms found.") + continue + + fig_json = go_data[go_type]["fig_json"] + df_dict = go_data[go_type]["df_dict"] + + fig = pio.from_json(fig_json) + + df_go = pd.DataFrame(df_dict) + + if df_go.empty: + st.info(f"No enriched {go_type} terms found.") + else: + st.plotly_chart(fig, width="stretch") + + st.markdown(f"#### {go_type} Enrichment Results") + st.dataframe(df_go, width="stretch") \ No newline at end of file diff --git a/content/results_pca.py b/content/results_pca.py index 45ea8eb..29f441a 100644 --- a/content/results_pca.py +++ b/content/results_pca.py @@ -1,11 +1,10 @@ """PCA Results Page.""" -import streamlit as st import pandas as pd -import plotly.express as px -from sklearn.decomposition import PCA -from sklearn.preprocessing import StandardScaler +import polars as pl +import streamlit as st from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import PCAPlot params = page_setup() st.title("PCA Analysis") @@ -13,7 +12,7 @@ st.markdown( """ Principal Component Analysis (PCA) of protein-level abundance. -Samples are colored by group assignment to visualize clustering. +Samples are projected onto their principal components and colored by group assignment to visualize clustering. """ ) @@ -21,6 +20,7 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Load abundance data (base wide-format table + sample -> group mapping) result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -28,60 +28,143 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +# Mirrors statistical.py: PCA should run on the most-processed data available. +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-adj", "stat", "p-value"] +] +unique_groups = sorted({sample_group_map[s] for s in sample_cols if s in sample_group_map}) + +if len(sample_cols) < 2: + st.info("PCA requires at least 2 samples.") + st.stop() -top_n = 500 +if len(unique_groups) < 2: + st.warning( + "Only one biological group was detected - points will still be plotted, " + "but group-based coloring requires 2 or more groups." + ) -top_proteins = ( - pivot_df - .dropna(subset=["p-adj"]) - .sort_values("p-adj", ascending=True) - .head(top_n)["ProteinName"] +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples " + f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})." ) +st.dataframe(base_df, use_container_width=True) -expr_df_pca = expr_df.loc[ - expr_df.index.intersection(top_proteins) -] +st.markdown("---") + +# --- SECTION 2: PCA Configuration --- +st.subheader("Configure PCA") + +expr_df_wide = base_df.set_index(id_col)[sample_cols] +max_available = expr_df_wide.shape[0] + +if max_available <= 20: + top_n = max_available + st.caption(f"Using all {top_n} proteins for PCA (dataset too small for variance filtering).") +else: + top_n = st.slider( + "Number of proteins (Highest Variance)", + min_value=20, + max_value=min(5000, max_available), + value=min(500, max_available), + step=10, + key="pca_top_n", + help=( + "PCA is computed only on the N proteins with the highest variance " + "across samples, to reduce noise from low-variance/uninformative features." + ), + ) + +top_proteins = expr_df_wide.var(axis=1).sort_values(ascending=False).head(top_n).index +expr_df_pca = expr_df_wide.loc[top_proteins].reset_index() if expr_df_pca.shape[0] < 2: - st.info("Not enough proteins after p-value filtering for PCA.") + st.info("Not enough proteins after variance filtering for PCA.") st.stop() -X = expr_df_pca.T -X_scaled = StandardScaler().fit_transform(X) - -pca = PCA(n_components=2) -pcs = pca.fit_transform(X_scaled) - -pca_df = pd.DataFrame( - pcs, - columns=["PC1", "PC2"], - index=X.index +# Prepare structural Polars metadata DataFrame required by PCAPlot +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, ) +pca_lazy = pl.from_pandas(expr_df_pca).lazy() + +# 3. Initialize the OpenMS-Insight PCAPlot component (computes PCA internally) +try: + pca_component = PCAPlot( + cache_id="quantms_pca_plot", + data=pca_lazy, + metadata=metadata_pl, + sample_id_field="sample_id", + group_field="group", + n_components=5, + title="Sample PCA", + ) +except ValueError as e: + st.error(f"PCA computation failed: {e}") + st.stop() -norm_map = { - k.replace(".mzML", ""): v - for k, v in group_map.items() -} -pca_df["Group"] = pca_df.index.map(norm_map) - -fig_pca = px.scatter( - pca_df, - x="PC1", - y="PC2", - color="Group", - text=pca_df.index, -) +variance_ratio = pca_component.get_variance_ratio() +pc_columns = pca_component.get_pc_columns() -fig_pca.update_traces(textposition="top center") -fig_pca.update_layout( - xaxis_title=f"PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)", - yaxis_title=f"PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)", - height=600, -) +# 4. Let the user pick which component pair to view (no recomputation needed) +col1, col2 = st.columns(2) +with col1: + pc_x_label = st.selectbox("X-axis component", pc_columns, index=0, key="pca_pc_x") +with col2: + default_y_index = 1 if len(pc_columns) > 1 else 0 + pc_y_label = st.selectbox("Y-axis component", pc_columns, index=default_y_index, key="pca_pc_y") + +pc_x = int(pc_x_label.replace("PC", "")) +pc_y = int(pc_y_label.replace("PC", "")) -st.plotly_chart(fig_pca, use_container_width=True) +# 5. Render the component +state_manager = st.session_state.get("state") +pca_component(state_manager=state_manager, pc_x=pc_x, pc_y=pc_y, height=600) -st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") +st.markdown( + "**Explained variance:** " + + ", ".join(f"{col} {ratio * 100:.1f}%" for col, ratio in zip(pc_columns, variance_ratio)) +) +st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by variance)") st.markdown("---") st.markdown("**Other visualizations:**") diff --git a/content/results_proteomicslfq.py b/content/results_proteomicslfq.py index 77eb332..fde2ab9 100644 --- a/content/results_proteomicslfq.py +++ b/content/results_proteomicslfq.py @@ -45,15 +45,14 @@ st.markdown("### 🧬 Protein-Level Abundance Table") st.info( "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." + "same protein and aggregating their intensities across samples." ) if pivot_df.empty: st.info("No protein-level data available.") else: st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df.sort_values("p-value"), use_container_width=True) + st.dataframe(pivot_df, use_container_width=True) # ====================================================== # GO Enrichment Results diff --git a/content/results_volcano.py b/content/results_volcano.py index 8502489..db2702f 100644 --- a/content/results_volcano.py +++ b/content/results_volcano.py @@ -1,9 +1,9 @@ """Volcano Plot Results Page.""" import streamlit as st -import plotly.express as px -import numpy as np +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column +from openms_insight import VolcanoPlot params = page_setup() st.title("Volcano Plot") @@ -19,6 +19,19 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Check if statistical analysis results are available in the session state +if "statistics_df" not in st.session_state or st.session_state["statistics_df"] is None: + st.info("Statistical analysis data not found. Please run the statistical engine first.") + st.page_link("content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬") + st.stop() + +# Retrieve the completed statistical analysis DataFrame +statistics_df = st.session_state["statistics_df"] + +if statistics_df.empty: + st.info("No data available for volcano plot.") + st.stop() + result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -26,16 +39,13 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) -if pivot_df.empty: - st.info("No data available for volcano plot.") - st.stop() - -volcano_df = pivot_df.copy() -volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"]) - -volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) +# 2. Clean data and convert to Polars for component input +volcano_df = statistics_df.dropna(subset=["log2FC", "p-adj"]).copy() +volcano_pl_lazy = pl.from_pandas(volcano_df).lazy() +# 3. Configure UI sliders (changing thresholds does not invalidate cache) fc_thresh = st.slider( "log2 Fold Change threshold", min_value=0.5, @@ -52,49 +62,34 @@ step=0.001, ) -volcano_df["Significance"] = "Not significant" -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", -] = "Up-regulated" - -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", -] = "Down-regulated" - -fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["ProteinName", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } +# 4. Initialize the OpenMS-Insight VolcanoPlot component +volcano_plot_component = VolcanoPlot( + cache_id="quantms_volcano_plot", + data=volcano_pl_lazy, + log2fc_column="log2FC", + pvalue_column="p-adj", + label_column=id_col, + up_color="#E74C3C", + down_color="#3498DB", + ns_color="#95A5A6", + show_threshold_lines=True, + threshold_line_style="dash", ) -fig_volcano.add_vline(x=fc_thresh, line_dash="dash") -fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") -fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - -# Make x-axis symmetric around zero -max_abs_fc = volcano_df["log2FC"].abs().max() -x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding +# 5. Render the component +state_manager = st.session_state.get("state") # Inject the project state management object -fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, +volcano_plot_component( + state_manager=state_manager, + fc_threshold=fc_thresh, + p_threshold=p_thresh, + max_labels=10, # Display labels for the top N significant proteins height=600, ) -st.plotly_chart(fig_volcano, use_container_width=True) - -up_count = (volcano_df["Significance"] == "Up-regulated").sum() -down_count = (volcano_df["Significance"] == "Down-regulated").sum() +# 6. Keep the existing statistical summary and bottom links +up_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh)).sum() +down_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh)).sum() st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") st.markdown("---") diff --git a/content/statistical.py b/content/statistical.py new file mode 100644 index 0000000..2e2a46d --- /dev/null +++ b/content/statistical.py @@ -0,0 +1,165 @@ +"""Statistical Inference Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import statistics engine functions from openms_insight +from openms_insight.analysis.statistics import calculate_statistical_tests, adjust_fdr_lazy + +params = page_setup() +st.title("Statistical Inference") + +st.markdown( + """ +Run differential expression analysis to identify statistically significant proteins across your biological groups. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract actual active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] +unique_groups = sorted(list(set([sample_group_map[s] for s in sample_cols if s in sample_group_map]))) +group_count = len(unique_groups) + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Dynamic Statistical Parameter Configuration --- +st.subheader("Configure Statistical Engine") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2 = st.columns(2) + +with col1: + st.markdown("### πŸ”¬ 1. Hypothesis Testing Method") + + # Route available method options dynamically based on the group count + if group_count == 2: + method_options = ["limma_like", "welch", "paired"] + help_text = "'limma_like' uses Empirical Bayes variance shrinking. 'welch' is for unequal variances. 'paired' is for dependent samples." + elif group_count >= 3: + method_options = ["limma_like", "anova"] + help_text = "'limma_like' supports multi-group design matrices. 'anova' computes standard row-wise One-way ANOVA." + else: + st.error("❌ Statistical testing requires at least 2 unique sample groups.") + st.stop() + + selected_method = st.selectbox( + "Select Statistical Test", + options=method_options, + index=0, + help=help_text + ) + +with col2: + st.markdown("### πŸ›‘οΈ 2. Multiple Testing Correction (FDR)") + selected_fdr = st.selectbox( + "Select FDR Adjustment Strategy", + options=["BH", "Bonferroni", "None"], + index=0, + help="'BH' (Benjamini-Hochberg) controls False Discovery Rate. 'Bonferroni' is strict Family-Wise Error Rate control." + ) + +# --- SECTION 3: Statistical Query Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Run Statistical Analysis", type="primary"): + + # Convert active pandas dataframe into polars lazyframe graph + stats_lazy = pl.from_pandas(base_df).lazy() + + try: + # Execute Chain 1: Calculate core statistics (Adds log2FC, stat, p-value) + stats_lazy = calculate_statistical_tests( + quantification_data=stats_lazy, + metadata=metadata_pl, + method=selected_method + ) + + # Execute Chain 2: Adjust Multiple Testing (Adds p-adj) + stats_lazy = adjust_fdr_lazy( + quantification_data=stats_lazy, + strategy=selected_fdr + ) + + # Resolve lazy graph optimization tree and bring back to pandas memory + statistics_df = stats_lazy.collect().to_pandas() + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (e.g., Volcano plot, Volcano/Heatmap UI) + st.session_state["statistics_df"] = statistics_df + + st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!") + + # Display the finalized statistics table view + st.subheader("Statistical Analysis Results") + st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`") + st.dataframe(statistics_df, use_container_width=True) + + except ValueError as val_err: + st.error(f"Engine Validation Fallure: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index aac2879..5d20693 100644 --- a/requirements.txt +++ b/requirements.txt @@ -142,6 +142,11 @@ scipy scikit-learn openms-insight>=0.1.13 polars>=1.0.0 +# Forces polars to prefer its most CPU-compatible native runtime. +# Without this, polars can select an AVX-optimized runtime (e.g. runtime-32) +# that access-violation crashes (0xC0000005 in _polars_runtime.pyd) on some +# CPUs (seen on AMD Threadripper PRO 3995WX on Windows) during dataframe ops. +polars-runtime-compat cython easypqp>=0.1.34 pyprophet>=2.2.0 @@ -149,4 +154,5 @@ mygene # Redis Queue dependencies (for online mode) redis>=5.0.0 rq>=1.16.0 -statsmodels \ No newline at end of file +statsmodels +polars \ No newline at end of file diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index 2af9bda..120efb0 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -1,9 +1,10 @@ import streamlit as st from pathlib import Path +import re import pandas as pd import plotly.express as px -#from streamlit_plotly_events import plotly_events -#from pyopenms import IdXMLFile +from streamlit_plotly_events import plotly_events +from pyopenms import IdXMLFile from scipy.stats import ttest_ind import numpy as np import mygene @@ -14,7 +15,7 @@ from src.common.common import page_setup from src.common.results_helpers import get_abundance_data from src.common.results_helpers import parse_idxml, build_spectra_cache -#from openms_insight import Table, Heatmap, LinePlot, SequenceView +from openms_insight import Table, Heatmap, LinePlot, SequenceView # params = page_setup() class WorkflowTest(WorkflowManager): @@ -47,6 +48,29 @@ def configure(self) -> None: self.ui.select_input_file("mzML-files", multiple=True, reactive=True) self.ui.select_input_file("fasta-file", multiple=False) + self.params = self.parameter_manager.get_parameters_from_json() + saved_mode = self.params.get("analysis-mode", "LFQ") + + self.ui.input_widget( + key="analysis-mode", + default=saved_mode, + name="Analysis Mode", + widget_type="selectbox", + options=["LFQ", "TMT"], + help="Choose between Label-Free Quantification (LFQ) or Tandem Mass Tag (TMT) analysis.", + reactive=True + ) + + self.params = self.parameter_manager.get_parameters_from_json() + current_mode = self.params.get("analysis-mode", "LFQ") + + if current_mode == "LFQ": + self.render_lfq_tabs() + else: + self.render_tmt_tabs() + + def render_lfq_tabs(self): + st.subheader("LFQ Analysis Mode") t = st.tabs(["**Identification**", "**Rescoring**", "**Filtering**", "**Library Generation**", "**Quantification**", "**Group Selection**"]) with t[0]: @@ -70,10 +94,10 @@ def configure(self) -> None: st.info(""" **Decoy Database Settings:** * **method**: How decoy sequences are generated from target protein sequences. - *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly - rearranges the amino acids. Both methods preserve the amino acid composition - of the original protein, ensuring decoys have similar properties to real sequences - for accurate false discovery rate (FDR) estimation. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. """) self.ui.input_TOPP( "DecoyDatabase", @@ -102,7 +126,7 @@ def configure(self) -> None: st.info(comet_info) comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] if not self.params.get("generate-decoys", True): # Only show decoy_string when not generating decoys comet_include.append("PeptideIndexing:decoy_string") @@ -111,7 +135,7 @@ def configure(self) -> None: "CometAdapter", custom_defaults={ "threads": 8, - "instrument": "high_res", + "instrument": "low_res", "missed_cleavages": 2, "min_peptide_length": 6, "max_peptide_length": 40, @@ -120,17 +144,19 @@ def configure(self) -> None: "isotope_error": "0/1", "precursor_charge": "2:4", "precursor_mass_tolerance": 20.0, - "fragment_mass_tolerance": 0.02, - "fragment_bin_offset": 0.0, + "fragment_mass_tolerance": 0.6, + "fragment_bin_offset": 0.4, "max_variable_mods_in_peptide": 3, "minimum_peaks": 1, "clip_nterm_methionine": "true", - "PeptideIndexing:IL_equivalent": "true", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)", + "PeptideIndexing:IL_equivalent": True, "PeptideIndexing:unmatched_action": "warn", "PeptideIndexing:decoy_string": "rev_", + "mass_recalibration": False, }, - flag_parameters=["PeptideIndexing:IL_equivalent"], include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "mass_recalibration"], exclude_parameters=["second_enzyme"], ) @@ -151,10 +177,10 @@ def configure(self) -> None: "subset_max_train": 300000, "decoy_pattern": "rev_", "score_type": "pep", - "post_processing_tdc": "true", + "post_processing_tdc": True, }, - flag_parameters=["post_processing_tdc"], include_parameters=percolator_include, + flag_parameters=["post_processing_tdc"], exclude_parameters=["out_type"], ) @@ -250,6 +276,10 @@ def configure(self) -> None: "psmFDR": 0.01, "proteinFDR": 0.01, "picked_proteinFDR": "true", + "alignment_order": "star", + "protein_quantification": "unique_peptides", + "quantification_method": "feature_intensity", + "protein_inference": "aggregation", }, include_parameters=["intThreshold", "psmFDR", "proteinFDR"], ) @@ -300,6 +330,294 @@ def configure(self) -> None: if orphaned_keys: self.parameter_manager.save_parameters() + def render_tmt_tabs(self): + st.subheader("TMT Analysis Mode") + # Create tabs for different analysis steps. + t = st.tabs( + ["**IsobaricAnalyzer**", "**CometAdapter**", "**PercolatorAdapter**", "**IDFilter**", "**IDMapper**", "**FileMerger**", + "**ProteinInference**", "**IDFilter**", "**IDConflictResolver**", "**ProteinQuantifier**", "**Group Selection**"] + ) + with t[0]: + # Checkbox for decoy generation + # reactive=True ensures the parent configure() fragment re-runs when checkbox changes, + # so conditional UI (DecoyDatabase settings) updates immediately + self.ui.input_widget( + key="generate-decoys", + default=True, + name="Generate Decoy Database", + widget_type="checkbox", + help="Generate reversed decoy sequences for FDR calculation. Disable if your FASTA already contains decoys.", + reactive=True, + ) + + # Reload params to get current checkbox value after it was saved + self.params = self.parameter_manager.get_parameters_from_json() + + # Show DecoyDatabase settings if generating decoys + if self.params.get("generate-decoys", True): + st.info(""" + **Decoy Database Settings:** + * **method**: How decoy sequences are generated from target protein sequences. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. + """) + self.ui.input_TOPP( + "DecoyDatabase", + custom_defaults={ + "decoy_string": "rev_", + "decoy_string_position": "prefix", + "method": "reverse", + }, + include_parameters=["method"], + ) + + comet_info = """ + **Identification (Comet):** + * **enzyme**: The enzyme used for peptide digestion. + * **missed_cleavages**: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. + * **fixed_modifications**: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **variable_modifications**: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **instrument**: Type of instrument (high_res or low_res). Use 'high_res' for high-resolution MS2 (Orbitrap, TOF), 'low_res' for ion trap. + * **fragment_mass_tolerance**: Fragment mass tolerance for MS2 matching. + * **fragment_bin_offset**: Offset for binning MS2 spectra. Typically 0.0 for high-res, 0.4 for low-res instruments. + """ + if not self.params.get("generate-decoys", True): + comet_info += """* **PeptideIndexing:decoy_string**: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions + in the protein database to indicate decoy proteins. + """ + st.info(comet_info) + + st.write(Path(self.workflow_dir, "results")) + + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + if not self.params.get("generate-decoys", True): + # Only show decoy_string when not generating decoys + comet_include.append("PeptideIndexing:decoy_string") + + self.ui.input_TOPP( + "IsobaricAnalyzer", + custom_defaults={ + "tmt11plex:reference_channel": 126, + "type": "tmt11plex", + "extraction:select_activation": "auto", + "extraction:reporter_mass_shift": 0.002, + "extraction:min_reporter_intensity": 0.0, + "extraction:min_precursor_purity": 0.0, + "extraction:precursor_isotope_deviation": 10.0, + "quantification:isotope_correction": "false", + }, + tool_instance_name="IsobaricAnalyzer-TMT", + reactive=True, + ) + with t[1]: + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset", "PeptideIndexing:IL_equivalent"] + self.ui.input_TOPP( + "CometAdapter", + custom_defaults={ + "PeptideIndexing:IL_equivalent": True, + "clip_nterm_methionine": "true", + "instrument": "high_res", + "missed_cleavages": 2, + "min_peptide_length": 6, + "max_peptide_length": 40, + "enzyme": "Trypsin/P", + "PeptideIndexing:unmatched_action": "warn", + "max_variable_mods_in_peptide": 3, + "precursor_mass_tolerance": 4.5, + "isotope_error": "0/1", + "precursor_error_units": "ppm", + "num_hits": 1, + "num_enzyme_termini": "fully", + "fragment_bin_offset": 0.0, + "minimum_peaks": 10, + "precursor_charge": "2:4", + "fragment_mass_tolerance": 0.015, + "PeptideIndexing:unmatched_action": "warn", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)\nTMT6plex (K)\nTMT6plex (N-term)", + "debug": 0, + "force": True, + }, + include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "force"], + exclude_parameters=["second_enzyme"], + tool_instance_name="CometAdapter-TMT", + ) + with t[2]: + st.info(""" + **Filtering (IDFilter):** + * **score:type_peptide**: Score used for filtering. If empty, the main score is used. + * **score:psm**: The score which should be reached by a peptide hit to be kept. (use 'NAN' to disable this filter) + """) + self.ui.input_TOPP( + "PercolatorAdapter", + custom_defaults={ + "subset_max_train": 300000, + "decoy_pattern": "DECOY_", + "score_type": "pep", + "post_processing_tdc": True, + "debug": 0, + }, + flag_parameters=["post_processing_tdc"], + tool_instance_name="PercolatorAdapter-TMT", + ) + + with t[3]: + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_peptide": "q-value", + "score:psm": 0.10, + }, + tool_instance_name="IDFilter-strict", + ) + with t[4]: + st.info(""" + **Quantification (ProteomicsLFQ):** + * **intThreshold**: Peak intensity threshold applied in seed detection. + * **psmFDR**: FDR threshold for sub-protein level (e.g. 0.05=5%). Use -FDR_type to choose the level. Cutoff is applied at the highest level. If Bayesian inference was chosen, it is equivalent with a peptide FDR + * **proteinFDR**: Protein FDR threshold (0.05=5%). + """) + self.ui.input_TOPP( + "IDMapper", + custom_defaults={ + "threads": 8, + "debug": 0, + }, + tool_instance_name="IDMapper-TMT", + ) + with t[5]: + self.ui.input_TOPP( + "FileMerger", + custom_defaults={ + "in_type": "consensusXML", + "append_method": "append_cols", + "annotate_file_origin": True, + "threads": 8, + }, + flag_parameters=["annotate_file_origin"], + tool_instance_name="FileMerger-TMT", + ) + with t[6]: + self.ui.input_TOPP( + "ProteinInference", + custom_defaults={ + "threads": 8, + "picked_decoy_string": "DECOY_", + "picked_fdr": "true", + "protein_fdr": "true", + "Algorithm:use_shared_peptides": "true", + "Algorithm:annotate_indistinguishable_groups": "true", + "Algorithm:score_type": "PEP", + "Algorithm:score_aggregation_method": "best", + "Algorithm:min_peptides_per_protein": 1, + }, + tool_instance_name="ProteinInference-TMT", + ) + with t[7]: + # A single checkbox widget for workflow logic. + # self.ui.input_widget("run-python-script", False, "Run custom Python script") * + # Generate input widgets for a custom Python tool, located at src/python-tools. + # Parameters are specified within the file in the DEFAULTS dictionary. + # self.ui.input_python("example") * + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_protein": "q-value", + "score:proteingroup": 0.01, + "score:psm": 0.01, + "delete_unreferenced_peptide_hits": True, + "remove_decoys": True + }, + flag_parameters=["delete_unreferenced_peptide_hits", "remove_decoys"], + tool_instance_name="IDFilter-lenient", + ) + with t[8]: + self.ui.input_TOPP( + "IDConflictResolver", + custom_defaults={ + "threads": 4, + }, + tool_instance_name="IDConflictResolver-TMT", + ) + + with t[9]: + self.ui.input_TOPP( + "ProteinQuantifier", + custom_defaults={ + "method": "top", + "top:N": 3, + "top:aggregate": "median", + "top:include_all": True, + "ratios": True, + "threads": 8, + "debug": 0, + }, + flag_parameters=["top:include_all", "ratios"], + tool_instance_name="ProteinQuantifier-TMT", + ) + with t[10]: + st.markdown("### πŸ§ͺ TMT Sample Group Assignment") + + latest_params = self.parameter_manager.get_parameters_from_json() + type_key = ( + f"{self.parameter_manager.topp_param_prefix}" + "IsobaricAnalyzer-TMT:1:type" + ) + selected_type = str( + st.session_state.get(type_key) + or latest_params.get("IsobaricAnalyzer-TMT", {}).get("type") + or "tmt11plex" + ).lower() + + m = re.search(r'\d+', selected_type) + is_supported_type = any(label in selected_type for label in ["tmt", "itraq"]) + if not m or not is_supported_type: + st.warning("Please select a supported isobaric type in the IsobaricAnalyzer tab first.") + else: + num_plex = int(m.group()) + channels = [f"sample{i+1}" for i in range(num_plex)] + st.caption(f"Isobaric type: **{selected_type}** - {num_plex} channels") + st.info("Assign a group name to each channel. Use **'skip'** to exclude a channel.") + + for row_start in range(0, num_plex, 2): + c1, c2 = st.columns(2) + + left_idx = row_start + left_channel = channels[left_idx] + with c1: + self.ui.input_widget( + key=f"TMT-group-{left_channel}", + default="", + name=f"Group for channel {left_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + right_idx = row_start + 1 + if right_idx < num_plex: + right_channel = channels[right_idx] + with c2: + self.ui.input_widget( + key=f"TMT-group-{right_channel}", + default="", + name=f"Group for channel {right_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + # Remove orphaned params from a previously selected larger plex + self.params = self.parameter_manager.get_parameters_from_json() + valid_keys = {f"TMT-group-{ch}" for ch in channels} + orphaned = [k for k in self.params if k.startswith("TMT-group-") and k not in valid_keys] + if orphaned: + for k in orphaned: + del self.params[k] + self.parameter_manager.save_parameters() + def execution(self) -> bool: """ Refactored TOPP workflow execution: @@ -352,639 +670,944 @@ def execution(self) -> bool: st.info(f"Using original FASTA: {fasta_path.name}") database_fasta = fasta_path - # ================================ - # 1️⃣ Directory setup - # ================================ - results_dir = Path(self.workflow_dir, "results") - comet_dir = results_dir / "comet_results" - perc_dir = results_dir / "percolator_results" - filter_dir = results_dir / "filter_results" - quant_dir = results_dir / "quant_results" - - for d in [comet_dir, perc_dir, filter_dir, quant_dir]: - d.mkdir(parents=True, exist_ok=True) - - self.logger.log("πŸ“ Output directories created") - - # # ================================ - # # 2️⃣ File path definitions (per sample) - # # ================================ - comet_results = [] - percolator_results = [] - filter_results = [] - - for mz in in_mzML: - stem = Path(mz).stem - comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) - percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) - filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + current_mode = self.params.get("analysis-mode", "LFQ") + st.write(f"Current analysis mode: **{current_mode}**") - # ================================ - # 3️⃣ Per-file processing - # ================================ - for i, mz in enumerate(in_mzML): - stem = Path(mz).stem - st.info(f"Processing sample: {stem}") + if current_mode == "LFQ": + self.logger.log("βš™οΈ Running LFQ workflow") - self.logger.log("πŸ”¬ Starting per-sample processing...") + # ================================ + # 1️⃣ Directory setup + # ================================ + results_dir = Path(self.workflow_dir, "results") + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + filter_dir = results_dir / "psm_filter" + quant_dir = results_dir / "quant_results" + + results_dir = Path(self.workflow_dir, "input-files") + + for d in [comet_dir, perc_dir, filter_dir, quant_dir]: + d.mkdir(parents=True, exist_ok=True) + + self.logger.log("πŸ“ Output directories created") + + # ================================ + # 2️⃣ File path definitions (per sample) + # ================================ + comet_results = [] + percolator_results = [] + filter_results = [] + + for mz in in_mzML: + stem = Path(mz).stem + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) + filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + + # ================================ + # 3️⃣ Per-file processing + # ================================ + for i, mz in enumerate(in_mzML): + stem = Path(mz).stem + st.info(f"Processing sample: {stem}") + + self.logger.log("πŸ”¬ Starting per-sample processing...") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - # --- CometAdapter --- - self.logger.log("πŸ”Ž Running peptide search...") - with st.spinner(f"CometAdapter ({stem})"): - comet_extra_params = {"database": str(database_fasta)} - if self.params.get("generate-decoys", True): - # Propagate decoy_string from DecoyDatabase - comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + ): + self.logger.log("Workflow stopped due to error") + return False - if not self.executor.run_topp( - "CometAdapter", - { - "in": in_mzML, - "out": comet_results, - }, - comet_extra_params, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Get fragment tolerance from CometAdapter parameters for visualization - comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") - frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) - frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" - - # Build visualization cache for Comet results - results_dir_path = Path(self.workflow_dir, "results") - cache_dir = results_dir_path / "insight_cache" - cache_dir.mkdir(parents=True, exist_ok=True) - - # Get mzML directory - mzml_dir = Path(in_mzML[0]).parent - - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} - - for idxml_file in comet_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - self.logger.log("βœ… Peptide search complete") + self.logger.log("βœ… Peptide search complete") - # --- PercolatorAdapter --- - self.logger.log("πŸ“Š Running rescoring...") - with st.spinner(f"PercolatorAdapter ({stem})"): - if not self.executor.run_topp( - "PercolatorAdapter", - { - "in": comet_results, - "out": percolator_results, - }, - {"decoy_pattern": decoy_string}, # Always propagated from upstream - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Percolator results - for idxml_file in percolator_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # if not Path(comet_results).exists(): + # st.error(f"CometAdapter failed for {stem}") + # st.stop() - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter ({stem})"): + if not self.executor.run_topp( + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + {"decoy_pattern": decoy_string}, # Always propagated from upstream + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - self.logger.log("βœ… Rescoring complete") + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # if not Path(percolator_results[i]).exists(): - # st.error(f"PercolatorAdapter failed for {stem}") - # st.stop() + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - # --- IDFilter --- - self.logger.log("πŸ”§ Filtering identifications...") - with st.spinner(f"IDFilter ({stem})"): - if not self.executor.run_topp( - "IDFilter", - { - "in": percolator_results, - "out": filter_results, - }, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Filter results - for idxml_file in filter_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + self.logger.log("βœ… Rescoring complete") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # if not Path(percolator_results[i]).exists(): + # st.error(f"PercolatorAdapter failed for {stem}") + # st.stop() - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter ({stem})"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": filter_results, + }, + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Build visualization cache for Filter results + for idxml_file in filter_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - self.logger.log("βœ… Filtering complete") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # if not Path(filter_results[i]).exists(): - # st.error(f"IDFilter failed for {stem}") - # st.stop() + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # ================================ - # EasyPQP Spectral Library Generation (optional) - # ================================ - if self.params.get("generate-library", False): - self.logger.log("πŸ“š Building spectral library with EasyPQP...") - st.info("Building spectral library with EasyPQP...") - library_dir = Path(self.workflow_dir, "results", "library") - library_dir.mkdir(parents=True, exist_ok=True) - - psms_files, peaks_files = [], [] - - for filter_idxml in filter_results: - original_stem = Path(filter_idxml).stem.replace("_filter", "") - matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) - if not matching_mzml: - self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") - continue - - # easypqp library requires specific extensions for file recognition: - # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension - # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension - # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name - psms_out = str(library_dir / f"{original_stem}.psmpkl") - peaks_out = str(library_dir / f"{original_stem}.peakpkl") - - convert_cmd = [ - "easypqp", "convert", - "--pepxml", filter_idxml, - "--spectra", matching_mzml, - "--psms", psms_out, - "--peaks", peaks_out - ] - if self.executor.run_command(convert_cmd): - psms_files.append(psms_out) - peaks_files.append(peaks_out) - - if psms_files: - # easypqp library outputs TSV format (despite common .pqp extension) - library_tsv = str(library_dir / "spectral_library.tsv") - library_cmd = ["easypqp", "library", "--out", library_tsv] - - if not self.params.get("library-use-fdr", False): - # --nofdr only skips FDR recalculation, NOT threshold filtering - # Set all thresholds to 1.0 to bypass filtering for pre-filtered input - library_cmd.extend([ - "--nofdr", - "--psm_fdr_threshold", "1.0", - "--peptide_fdr_threshold", "1.0", - "--protein_fdr_threshold", "1.0" - ]) - else: - # Apply user-specified FDR filtering - library_cmd.extend([ - "--psm_fdr_threshold", - str(self.params.get("library-psm-fdr", 0.01)), - "--peptide_fdr_threshold", - str(self.params.get("library-peptide-fdr", 0.01)), - "--protein_fdr_threshold", - str(self.params.get("library-protein-fdr", 0.01)) - ]) - - for psms, peaks in zip(psms_files, peaks_files): - library_cmd.extend([psms, peaks]) - - if self.executor.run_command(library_cmd): - self.logger.log("βœ… Spectral library created") - st.success("Spectral library created") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + self.logger.log("βœ… Filtering complete") + + # if not Path(filter_results[i]).exists(): + # st.error(f"IDFilter failed for {stem}") + # st.stop() + + # ================================ + # EasyPQP Spectral Library Generation (optional) + # ================================ + if self.params.get("generate-library", False): + self.logger.log("πŸ“š Building spectral library with EasyPQP...") + st.info("Building spectral library with EasyPQP...") + library_dir = Path(self.workflow_dir, "results", "library") + library_dir.mkdir(parents=True, exist_ok=True) + + psms_files, peaks_files = [], [] + + for filter_idxml in filter_results: + original_stem = Path(filter_idxml).stem.replace("_filter", "") + matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) + if not matching_mzml: + self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") + continue + + # easypqp library requires specific extensions for file recognition: + # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension + # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension + # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name + psms_out = str(library_dir / f"{original_stem}.psmpkl") + peaks_out = str(library_dir / f"{original_stem}.peakpkl") + + convert_cmd = [ + "easypqp", "convert", + "--pepxml", filter_idxml, + "--spectra", matching_mzml, + "--psms", psms_out, + "--peaks", peaks_out + ] + if self.executor.run_command(convert_cmd): + psms_files.append(psms_out) + peaks_files.append(peaks_out) + + if psms_files: + # easypqp library outputs TSV format (despite common .pqp extension) + library_tsv = str(library_dir / "spectral_library.tsv") + library_cmd = ["easypqp", "library", "--out", library_tsv] + + if not self.params.get("library-use-fdr", False): + # --nofdr only skips FDR recalculation, NOT threshold filtering + # Set all thresholds to 1.0 to bypass filtering for pre-filtered input + library_cmd.extend([ + "--nofdr", + "--psm_fdr_threshold", "1.0", + "--peptide_fdr_threshold", "1.0", + "--protein_fdr_threshold", "1.0" + ]) + else: + # Apply user-specified FDR filtering + library_cmd.extend([ + "--psm_fdr_threshold", + str(self.params.get("library-psm-fdr", 0.01)), + "--peptide_fdr_threshold", + str(self.params.get("library-peptide-fdr", 0.01)), + "--protein_fdr_threshold", + str(self.params.get("library-protein-fdr", 0.01)) + ]) + + for psms, peaks in zip(psms_files, peaks_files): + library_cmd.extend([psms, peaks]) + + if self.executor.run_command(library_cmd): + self.logger.log("βœ… Spectral library created") + st.success("Spectral library created") + else: + self.logger.log("Warning: Failed to build spectral library") else: - self.logger.log("Warning: Failed to build spectral library") - else: - self.logger.log("Warning: No PSMs converted for library generation") + self.logger.log("Warning: No PSMs converted for library generation") + + st.success(f"βœ“ {stem} identification completed") + + # # ================================ + # # 4️⃣ ProteomicsLFQ (cross-sample) + # # ================================ + self.logger.log("πŸ“ˆ Running cross-sample quantification...") + st.info("Running ProteomicsLFQ (cross-sample quantification)") + + quant_mztab = str(quant_dir / "openms_quant.mzTab") + quant_cxml = str(quant_dir / "openms.consensusXML") + quant_msstats = str(quant_dir / "openms_msstats.csv") + + with st.spinner("ProteomicsLFQ"): + combined_in = " ".join(in_mzML) + combined_ids = " ".join(filter_results) + self.logger.log(f"COMBINED_IN {combined_in}", 1) + self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) + self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) + self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + + # βœ… Streamlit output (debug view) + st.markdown("### πŸ” ProteomicsLFQ Input Debug") + st.write("**combined_in:**", combined_in) + st.write("**combined_in type:**", type(combined_in).__name__) + + st.write("**combined_ids:**", combined_ids) + st.write("**combined_ids type:**", type(combined_ids).__name__) + + if not self.executor.run_topp( + "ProteomicsLFQ", + { + "in": [in_mzML], + "ids": [filter_results], + "out": [quant_mztab], + "out_cxml": [quant_cxml], + "out_msstats": [quant_msstats], + }, + { + "fasta": str(database_fasta), + "threads": 12, + # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 + "PeptideQuantification:extract:IM_window": "0.0", + "PeptideQuantification:faims:merge_features": "false", + }, + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… Quantification complete") + + # if not Path(quant_mztab).exists(): + # st.error("ProteomicsLFQ failed: mzTab not created") + # st.stop() + + # ================================ + # 5️⃣ Final report + # # ================================ + st.success("πŸŽ‰ TOPP workflow completed successfully") + st.write("πŸ“ Results directory:") + st.code(str(results_dir)) + + st.write("πŸ“„ Generated files:") + st.write(f"- mzTab: {quant_mztab}") + st.write(f"- consensusXML: {quant_cxml}") + st.write(f"- MSstats CSV: {quant_msstats}") - st.success(f"βœ“ {stem} identification completed") + return True + else: + self.logger.log("βš™οΈ Running TMT workflow") - # ================================ - # 4️⃣ ProteomicsLFQ (cross-sample) - # ================================ - self.logger.log("πŸ“ˆ Running cross-sample quantification...") - st.info("Running ProteomicsLFQ (cross-sample quantification)") + results_dir = Path(self.workflow_dir, "results") + iso_dir = results_dir / "isobaric_consensusXML" + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + psm_filter_dir = results_dir / "psm_filter" + map_dir = results_dir / "idmapper" + merge_dir = results_dir / "merged" + protein_dir = results_dir / "protein" + msstats_dir = results_dir / "msstats" + quant_dir = results_dir / "quant_results" + + iso_consensus = [] + comet_results = [] + percolator_results = [] + psm_filtered = [] + mapped_ids = [] + + for d in [ + iso_dir, comet_dir, perc_dir, psm_filter_dir, + map_dir, merge_dir, protein_dir, msstats_dir, quant_dir + ]: + d.mkdir(parents=True, exist_ok=True) + + for mz in in_mzML: + stem = Path(mz).stem + iso_consensus.append(str(iso_dir / f"{stem}_iso.consensusXML")) + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_comet_perc.idXML")) + psm_filtered.append(str(psm_filter_dir / f"{stem}_comet_perc_filter.idXML")) + mapped_ids.append(str(map_dir / f"{stem}_comet_perc_filter_map.consensusXML")) + + merged_id = str(merge_dir / "ID_mapper_merge.consensusXML") + protein_id = str(protein_dir / "ID_mapper_merge_epi.consensusXML") + protein_filter = str(protein_dir / "ID_mapper_merge_epi_filter.consensusXML") + protein_resolved = str(protein_dir / "ID_mapper_merge_epi_filter_resconf.consensusXML") + consensus_out = str(quant_dir / "openms_design_protein_openms.csv") + + # --- IsobaricAnalyzer --- + self.logger.log("🏷️ Running isobaric labeling analysis...") + with st.spinner("IsobaricAnalyzer"): + if not self.executor.run_topp( + "IsobaricAnalyzer", + { + "in": in_mzML, + "out": iso_consensus, + }, + tool_instance_name="IsobaricAnalyzer-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IsobaricAnalyzer complete") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + tool_instance_name="CometAdapter-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… CometAdapter complete") + + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - quant_mztab = str(quant_dir / "openms_quant.mzTab") - quant_cxml = str(quant_dir / "openms.consensusXML") - quant_msstats = str(quant_dir / "openms_msstats.csv") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - with st.spinner("ProteomicsLFQ"): - combined_in = " ".join(in_mzML) - combined_ids = " ".join(filter_results) - self.logger.log(f"COMBINED_IN {combined_in}", 1) - self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) - self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) - self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # βœ… Streamlit output (debug view) - st.markdown("### πŸ” ProteomicsLFQ Input Debug") - st.write("**combined_in:**", combined_in) - st.write("**combined_in type:**", type(combined_in).__name__) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - st.write("**combined_ids:**", combined_ids) - st.write("**combined_ids type:**", type(combined_ids).__name__) + self.logger.log("βœ… Peptide search complete") + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter"): if not self.executor.run_topp( - "ProteomicsLFQ", - { - "in": [in_mzML], - "ids": [filter_results], - "out": [quant_mztab], - "out_cxml": [quant_cxml], - "out_msstats": [quant_msstats], - }, - { - "fasta": str(database_fasta), - "psmFDR": 0.5, - "proteinFDR": 0.5, - "threads": 12, - # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 - "PeptideQuantification:extract:IM_window": "0.0", - "PeptideQuantification:faims:merge_features": "false", - } - ): + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + tool_instance_name="PercolatorAdapter-TMT", + ): self.logger.log("Workflow stopped due to error") return False - self.logger.log("βœ… Quantification complete") - - # ====================================================== - # ⚠️ 5️⃣ GO Enrichment Analysis (INLINE IN EXECUTION) - # ====================================================== - workspace_path = Path(self.workflow_dir).parent - res = get_abundance_data(workspace_path) - if res is not None: - pivot_df, _, _ = res - self.logger.log("βœ… pivot_df loaded, starting GO enrichment...") - self._run_go_enrichment(pivot_df, results_dir) - else: - st.warning("GO enrichment skipped: abundance data not available.") + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # ================================ - # 5️⃣ Final report - # # ================================ - st.success("πŸŽ‰ TOPP workflow completed successfully") - st.write("πŸ“ Results directory:") - st.code(str(results_dir)) - - return True - - def _run_go_enrichment(self, pivot_df: pd.DataFrame, results_dir: Path): - p_cutoff = 0.05 - fc_cutoff = 1.0 - - analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() - - if analysis_df.empty: - st.error("No valid statistical data found for GO enrichment.") - self.logger.log("❗ analysis_df is empty") - else: - with st.spinner("Fetching GO terms from MyGene.info API..."): - mg = mygene.MyGeneInfo() - - def get_clean_uniprot(name): - parts = str(name).split("|") - return parts[1] if len(parts) >= 2 else parts[0] - - analysis_df["UniProt"] = analysis_df["ProteinName"].apply(get_clean_uniprot) - - bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() - fg_ids = analysis_df[ - (analysis_df["p-value"] < p_cutoff) & - (analysis_df["log2FC"].abs() >= fc_cutoff) - ]["UniProt"].dropna().astype(str).unique().tolist() - self.logger.log("βœ… get_clean_uniprot applied") - - if len(fg_ids) < 3: - st.warning( - f"Not enough significant proteins " - f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " - f"Found: {len(fg_ids)}" - ) - self.logger.log("❗ Not enough significant proteins") - else: - res_list = mg.querymany( - bg_ids, scopes="uniprot", fields="go", as_dataframe=False - ) - res_go = pd.DataFrame(res_list) - if "notfound" in res_go.columns: - res_go = res_go[res_go["notfound"] != True] - - def extract_go_terms(go_data, go_type): - if not isinstance(go_data, dict) or go_type not in go_data: - return [] - terms = go_data[go_type] - if isinstance(terms, dict): - terms = [terms] - return list({t.get("term") for t in terms if "term" in t}) - - for go_type in ["BP", "CC", "MF"]: - res_go[f"{go_type}_terms"] = res_go["go"].apply( - lambda x: extract_go_terms(x, go_type) - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - annotated_ids = set(res_go["query"].astype(str)) - fg_set = annotated_ids.intersection(fg_ids) - bg_set = annotated_ids - self.logger.log(f"βœ… fg_set bg_set are set") - - def run_go(go_type): - go2fg = defaultdict(set) - go2bg = defaultdict(set) - - for _, row in res_go.iterrows(): - uid = str(row["query"]) - for term in row[f"{go_type}_terms"]: - go2bg[term].add(uid) - if uid in fg_set: - go2fg[term].add(uid) - - records = [] - N_fg = len(fg_set) - N_bg = len(bg_set) - - for term, fg_genes in go2fg.items(): - a = len(fg_genes) - if a == 0: - continue - b = N_fg - a - c = len(go2bg[term]) - a - d = N_bg - (a + b + c) - - _, p = fisher_exact([[a, b], [c, d]], alternative="greater") - records.append({ - "GO_Term": term, - "Count": a, - "GeneRatio": f"{a}/{N_fg}", - "p_value": p, - }) - - df = pd.DataFrame(records) - if df.empty: - return None, None - - df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) - df = df.sort_values("p_value").head(20) - - # βœ… Plotly Figure - fig = px.bar( - df, - x="-log10(p)", - y="GO_Term", - orientation="h", - title=f"GO Enrichment ({go_type})", - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - self.logger.log(f"βœ… Plotly Figure generated") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - fig.update_layout( - yaxis=dict(autorange="reversed"), - height=500, - margin=dict(l=10, r=10, t=40, b=10), - ) + self.logger.log("βœ… PercolatorAdapter complete") + + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": psm_filtered, + }, + tool_instance_name="IDFilter-strict" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-strict complete") + + # Build visualization cache for Filter results + for idxml_file in psm_filtered: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - return fig, df + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - go_results = {} + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - for go_type in ["BP", "CC", "MF"]: - fig, df_go = run_go(go_type) - if fig is not None: - go_results[go_type] = { - "fig": fig, - "df": df_go - } - self.logger.log(f"βœ… go_type generated") - - go_dir = results_dir / "go-terms" - go_dir.mkdir(parents=True, exist_ok=True) - - import json - go_data = {} - - for go_type in ["BP", "CC", "MF"]: - if go_type in go_results: - fig = go_results[go_type]["fig"] - df = go_results[go_type]["df"] - - go_data[go_type] = { - "fig_json": fig.to_json(), # Figure β†’ JSON string - "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts - } - - go_json_file = go_dir / "go_results.json" - with open(go_json_file, "w") as f: - json.dump(go_data, f) - st.session_state["go_results"] = go_results - st.session_state["go_ready"] = True if go_data else False - self.logger.log("βœ… GO enrichment analysis complete") - + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + # --- IDMapper --- + self.logger.log("πŸ—ΊοΈ Mapping IDs to isobaric consensus features...") + for iso, psm, mapped in zip(iso_consensus, psm_filtered, mapped_ids): + iso_stem = Path(iso).stem + with st.spinner(f"IDMapper ({iso_stem})"): + if not self.executor.run_topp( + "IDMapper", + { + "in": [iso], + "id": [psm], + "out": [mapped], + }, + tool_instance_name="IDMapper-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDMapper complete") + + # --- FileMerger --- + self.logger.log("πŸ”— Merging mapped consensus files...") + with st.spinner("FileMerger"): + if not self.executor.run_topp( + "FileMerger", + { + "in": mapped_ids, + "out": [merged_id], + }, + tool_instance_name="FileMerger-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… FileMerger complete") + + # --- ProteinInference --- + self.logger.log("🧩 Running protein inference...") + with st.spinner("ProteinInference"): + if not self.executor.run_topp( + "ProteinInference", + { + "in": [merged_id], + "out": [protein_id], + }, + tool_instance_name="ProteinInference-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinInference complete") + + # --- IDFilter-lenient (Protein) --- + self.logger.log("πŸ”¬ Filtering proteins...") + with st.spinner("IDFilter (Protein)"): + if not self.executor.run_topp( + "IDFilter", + { + "in": [protein_id], + "out": [protein_filter], + }, + tool_instance_name="IDFilter-lenient" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-lenient (Protein) complete") + + # ================================ + # ✨ NEW: 8️⃣ IDConflictResolver (protein_filter β†’ protein_resolved) + # ================================ + self.logger.log("βš–οΈ Resolving ID conflicts...") + with st.spinner("IDConflictResolver"): + if not self.executor.run_topp( + "IDConflictResolver", + { + "in": [protein_filter], + "out": [protein_resolved], + }, + tool_instance_name="IDConflictResolver-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDConflictResolver complete") + + # ================================ + # ✨ NEW: πŸ”Ÿ ProteinQuantifier (protein_resolved β†’ consensus_out) + # ================================ + self.logger.log("πŸ“ Running protein quantification...") + with st.spinner("ProteinQuantifier"): + if not self.executor.run_topp( + "ProteinQuantifier", + { + "in": [protein_resolved], + "out": [consensus_out], + }, + tool_instance_name="ProteinQuantifier-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinQuantifier complete") + self.logger.log("πŸ“„ Generating protein table...") + + self.logger.log("πŸŽ‰ WORKFLOW FINISHED") @st.fragment def results(self) -> None: diff --git a/src/common/results_helpers.py b/src/common/results_helpers.py index db3e103..2d38ad9 100644 --- a/src/common/results_helpers.py +++ b/src/common/results_helpers.py @@ -5,10 +5,8 @@ import numpy as np import streamlit as st from pathlib import Path -from scipy.stats import ttest_ind from pyopenms import IdXMLFile, MSExperiment, MzMLFile from src.workflow.ParameterManager import ParameterManager -from statsmodels.stats.multitest import multipletests def get_workflow_dir(workspace): """Get the workflow directory path.""" @@ -184,12 +182,15 @@ def build_spectra_cache(mzml_dir: Path, filename_to_index: dict) -> tuple[pl.Dat @st.cache_data -def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: - """Load CSV, compute stats (log2FC, p-value), build pivot_df and expr_df. +def load_abundance_data(workspace_path: str, csv_mtime: float, params_mtime: float = 0.0) -> tuple | None: + """Load CSV and build abundance matrices for downstream preprocessing. Args: workspace_path: Path to the workspace directory csv_mtime: Modification time of CSV file (used as cache key) + params_mtime: Modification time of params.json (used as cache key so + changing group assignments in Configure invalidates the cache + even when the CSV itself hasn't changed) Returns: Tuple of (pivot_df, expr_df, group_map) or None if data unavailable @@ -197,115 +198,166 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: workflow_dir = get_workflow_dir(Path(workspace_path)) quant_dir = workflow_dir / "results" / "quant_results" - if not quant_dir.exists(): - return None - - csv_files = sorted(quant_dir.glob("*.csv")) - if not csv_files: - return None - - csv_file = csv_files[0] - - try: - df = pd.read_csv(csv_file) - except Exception: - return None + parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - if df.empty: - return None + workflow_params = parameter_manager.get_parameters_from_json() + analysis_mode = workflow_params.get("analysis-mode", "LFQ") - # Get group mapping from parameters - param_manager = ParameterManager(workflow_dir) - params = param_manager.get_parameters_from_json() - group_map = { - key[11:]: value # Remove "mzML-group-" prefix - for key, value in params.items() - if key.startswith("mzML-group-") and value - } + if analysis_mode == "LFQ": + if not quant_dir.exists(): + return None - if not group_map: - return None + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None - df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) - df["Group"] = df["Reference"].map(group_map) - df = df.dropna(subset=["Group"]) + csv_file = csv_files[0] - groups = sorted(df["Group"].unique()) + try: + df = pd.read_csv(csv_file) + except Exception: + return None - if len(groups) < 2: - return None + if df.empty: + return None - group1, group2 = groups[:2] - - # Compute statistics per protein - stats_rows = [] - for protein, protein_df in df.groupby("ProteinName"): - g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values - g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values + # Get optional group mapping from parameters. + # Group information is not required at this stage; statistical testing + # happens in the Statistical page. + param_manager = ParameterManager(workflow_dir) + params = param_manager.get_parameters_from_json() + group_map = { + key[11:]: value # Remove "mzML-group-" prefix + for key, value in params.items() + if key.startswith("mzML-group-") and value + } - if len(g1_vals) < 2 or len(g2_vals) < 2: - pval = np.nan + df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) + + # Build sample display order. + if group_map: + sample_group_df = df[["Sample", "Reference"]].drop_duplicates() + sample_group_df["Group"] = sample_group_df["Reference"].map(group_map) + grouped_samples = [] + for grp in sorted(sample_group_df["Group"].dropna().unique()): + grouped_samples.extend( + sample_group_df[sample_group_df["Group"] == grp]["Sample"].tolist() + ) + remaining_samples = [ + s for s in sorted(df["Sample"].unique()) if s not in grouped_samples + ] + all_samples = grouped_samples + remaining_samples else: - _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False) - - mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan - mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan - - log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan + all_samples = sorted(df["Sample"].unique()) + + # Build pivot table + pivot_list = [] + for protein, group_df in df.groupby("ProteinName"): + peptides = ";".join(group_df["PeptideSequence"].unique()) + intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() + intensity_dict_complete = { + sample: intensity_dict.get(sample, 0) + for sample in all_samples + } + row = { + "ProteinName": protein, + **intensity_dict_complete, + "PeptideSequence": peptides, + } + pivot_list.append(row) + + pivot_df = pd.DataFrame(pivot_list) + pivot_df = pivot_df[["ProteinName"] + all_samples + ["PeptideSequence"]] + + # Build expression matrix (log2-transformed) + expr_df = pivot_df.set_index("ProteinName")[all_samples] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() + + return pivot_df, expr_df, group_map + + else: + if not quant_dir.exists(): + return None + + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None + + csv_file = csv_files[0] + + try: + df = pd.read_csv(csv_file, sep="\t", comment="#", engine="python") + except Exception: + return None + + if df.empty: + return None + + # ratio column removal + df = df.loc[:, ~df.columns.str.contains('ratio', case=False)] + + # exclude_indices = st.session_state.get("tmt_exclude_indices", []) + # group_map = st.session_state.get("tmt_group_map", {}) + # Get group mapping from parameters + parameter_manager = ParameterManager(Path(workflow_dir), "TOPP Workflow") + params = parameter_manager.get_parameters_from_json() + group_map = {} + for key, value in params.items(): + if key.startswith("TMT-group-") and value: + # Extract the numeric part from keys like "TMT-group-sample1" + match = re.search(r'sample(\d+)', key) + if match: + # Subtract 1 to convert to a 0-based index (0, 1, 2...). + # If your samples are already 0-based, remove the -1 adjustment. + index = str(int(match.group(1)) - 1) + group_map[index] = value + + # 1. Extract keys labeled as "skip" from group_map as integer list + exclude_indices = [ + int(k) for k, v in group_map.items() if v.lower() == "skip" + ] + + # 2. Remove "skip" entries from group_map (keep only actual group info) + group_map = { + int(k): v for k, v in group_map.items() if v.lower() != "skip" + } - stats_rows.append({ - "ProteinName": protein, - "log2FC": log2fc, - "p-value": pval, - }) + start_column_offset = 4 - stats_df = pd.DataFrame(stats_rows) + # st.write("exclude_indices:", exclude_indices) + # st.write("group_map:", group_map) - if not stats_df.empty: - mask = stats_df["p-value"].notna() - if mask.any(): - _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh") - stats_df.loc[mask, "p-adj"] = p_adj + if exclude_indices: + # st.write("Current columns:", df.columns.tolist()) + # st.write("Number of columns:", len(df.columns)) + # st.write("Exclude indices:", exclude_indices) + # st.write("Offset:", start_column_offset) + cols_to_drop = [df.columns[i + start_column_offset] for i in exclude_indices] + df_cleaned = df.drop(columns=cols_to_drop) else: - stats_df["p-adj"] = np.nan - - # Order samples by group (group2 first, then group1) - sample_group_df = df[["Sample", "Group"]].drop_duplicates() - group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist() - group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist() - all_samples = group2_samples + group1_samples - - # Build pivot table - pivot_list = [] - for protein, group_df in df.groupby("ProteinName"): - peptides = ";".join(group_df["PeptideSequence"].unique()) - intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() - intensity_dict_complete = { - sample: intensity_dict.get(sample, 0) - for sample in all_samples - } - row = { - "ProteinName": protein, - **intensity_dict_complete, - "PeptideSequence": peptides, - } - pivot_list.append(row) + df_cleaned = df.copy() + + current_cols = df_cleaned.columns.tolist() + sample_cols = current_cols[start_column_offset:] - pivot_df = pd.DataFrame(pivot_list) - pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left") - pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]] + # Ensure sample columns are numeric for downstream preprocessing/statistics. + pivot_df = df_cleaned.copy() + if sample_cols: + pivot_df[sample_cols] = pivot_df[sample_cols].apply(pd.to_numeric, errors='coerce') - # Build expression matrix (log2-transformed) - expr_df = pivot_df.set_index("ProteinName")[all_samples] - expr_df = expr_df.replace(0, np.nan) - expr_df = np.log2(expr_df + 1) - expr_df = expr_df.dropna() + protein_col = pivot_df.columns[0] + expr_df = pivot_df.set_index(protein_col)[sample_cols] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() - return pivot_df, expr_df, group_map + return pivot_df, expr_df, group_map def get_abundance_data(workspace: Path) -> tuple | None: - """Wrapper that handles cache key (workspace + CSV mtime). + """Wrapper that handles cache key (workspace + CSV mtime + params mtime). Args: workspace: Path to the workspace directory @@ -324,4 +376,49 @@ def get_abundance_data(workspace: Path) -> tuple | None: return None csv_mtime = csv_files[0].stat().st_mtime - return load_abundance_data(str(workspace), csv_mtime) + + params_file = workflow_dir / "params.json" + params_mtime = params_file.stat().st_mtime if params_file.exists() else 0.0 + + return load_abundance_data(str(workspace), csv_mtime, params_mtime) + + +def get_id_column(workspace: Path, pivot_df: pd.DataFrame) -> str: + """Resolve the protein/row identifier column for the active analysis mode. + + LFQ reports always use "ProteinName"; TMT reports use whatever the + report's first column is actually named (e.g. "protein"). + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + return "ProteinName" if analysis_mode == "LFQ" else pivot_df.columns[0] + + +def get_sample_group_map(workspace: Path, pivot_df: pd.DataFrame, group_map: dict) -> dict: + """Normalize group_map into {actual_sample_column_name: group_name}. + + LFQ group_map keys are already clean sample names (optionally with a + ".mzML" suffix). TMT group_map keys are 0-based channel indices that must + be matched against the report's actual "sampleN[...]" column names. + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + + if analysis_mode == "LFQ": + return { + k[:-5] if k.endswith(".mzML") else k: v + for k, v in group_map.items() + } + + actual_sample_names = pivot_df.columns.tolist() + norm_map = {} + for k, v in group_map.items(): + try: + sample_idx = int(k) + 1 + except (TypeError, ValueError): + continue + target_substring = f"sample{sample_idx}[" + real_full_name = next((name for name in actual_sample_names if target_substring in name), None) + if real_full_name: + norm_map[real_full_name] = v if v and v.strip() else "Unassigned" + return norm_map From 4a2792a307d08f4f211075a0753916d45ebcd8d7 Mon Sep 17 00:00:00 2001 From: Yoo HoJun Date: Thu, 13 Aug 2026 14:54:26 +0900 Subject: [PATCH 08/10] Replace with streamlit-template files, keeping content/, k8s/, entrypoint.sh, app.py, src/Workflow.py Wholesale swap-in of streamlit-template's current files for every path except the quantms-web-specific ones (content pages, k8s manifests, the container entrypoint, app.py's navigation, and the template's example Workflow.py). Files that exist only in quantms-web (e.g. src/WorkflowTest.py, src/workflow/QueueManager.py, default-parameters.json values) have no template counterpart to replace them with, so they are left untouched. requirements.txt is the one file merged rather than replaced outright: template's dependency list is the base, with quantms-web-only packages (openms-insight, easypqp, pyprophet, mygene, polars, scipy, scikit-learn, streamlit_plotly_events, statsmodels, cython) added back so the app still runs. default-parameters.json, presets.json, and settings.json were restored to their quantms-web values after the initial replace overwrote quantms-specific defaults. Co-Authored-By: Claude Sonnet 5 --- .claude/skills/create-workflow.md | 22 + .github/workflows/build-and-test.yml | 570 +++++++++++++++++- .../build-windows-executable-app.yaml | 8 +- .github/workflows/ci.yml | 6 +- .github/workflows/ghcr-cleanup.yml | 26 + .../workflows/test-win-exe-w-embed-py.yaml | 44 +- .../workflows/test-win-exe-w-pyinstaller.yaml | 5 +- .github/workflows/workflow-tests.yml | 28 + .gitignore | 1 + .streamlit/config.toml | 3 +- CLAUDE.md | 199 ++++-- Dockerfile | 112 ++-- Dockerfile.arm | 237 ++++++++ Dockerfile_simple | 127 ++++ Dockerfile_simple.arm | 127 ++++ README.md | 230 +++++-- docker-compose.yml | 5 + docker/entrypoint.sh | 224 +++++++ docs/build_app.md | 137 +++++ docs/deployment.md | 100 +++ docs/installation.md | 109 ++++ docs/toppframework.py | 348 +++++++++++ docs/user_guide.md | 77 +++ docs/win_exe_with_embed_py.md | 278 +++++++++ docs/win_exe_with_pyinstaller.md | 140 +++++ gdpr_consent/dist/bundle.js | 2 +- gdpr_consent/src/main.ts | 13 + requirements.txt | 22 +- src/common/admin.py | 3 +- src/common/captcha_.py | 11 +- src/common/common.py | 48 +- src/mzmlfileworkflow.py | 107 ++++ src/peptide_mz_calculator.py | 107 ++++ src/python-tools/example.py | 67 ++ .../export_consensus_feature_df.py | 46 ++ src/run_subprocess.py | 57 ++ src/simpleworkflow.py | 13 + src/view.py | 327 ++++++++++ src/workflow/CommandExecutor.py | 15 +- src/workflow/StreamlitUI.py | 99 ++- src/workflow/WorkflowManager.py | 16 +- test.py | 24 + test_gui.py | 157 ++++- tests/test_legal_links.py | 160 +++++ tests/test_parameter_defaults.py | 363 +++++++++++ tests/test_queue_manager_cancel.py | 4 +- tests/test_run_subprocess.py | 37 ++ tests/test_simple_workflow.py | 69 +++ tests/test_tool_instance_name.py | 268 ++++++++ tests/test_topp_flag_parameters.py | 346 +++++++++++ 50 files changed, 5187 insertions(+), 357 deletions(-) create mode 100644 .github/workflows/workflow-tests.yml create mode 100644 Dockerfile.arm create mode 100644 Dockerfile_simple create mode 100644 Dockerfile_simple.arm create mode 100755 docker/entrypoint.sh create mode 100644 docs/build_app.md create mode 100644 docs/deployment.md create mode 100644 docs/installation.md create mode 100644 docs/toppframework.py create mode 100644 docs/user_guide.md create mode 100644 docs/win_exe_with_embed_py.md create mode 100644 docs/win_exe_with_pyinstaller.md create mode 100644 src/mzmlfileworkflow.py create mode 100644 src/peptide_mz_calculator.py create mode 100644 src/python-tools/example.py create mode 100644 src/python-tools/export_consensus_feature_df.py create mode 100644 src/run_subprocess.py create mode 100644 src/simpleworkflow.py create mode 100644 src/view.py create mode 100644 test.py create mode 100644 tests/test_legal_links.py create mode 100644 tests/test_parameter_defaults.py create mode 100644 tests/test_run_subprocess.py create mode 100644 tests/test_simple_workflow.py create mode 100644 tests/test_tool_instance_name.py create mode 100644 tests/test_topp_flag_parameters.py diff --git a/.claude/skills/create-workflow.md b/.claude/skills/create-workflow.md index 07856db..38e981d 100644 --- a/.claude/skills/create-workflow.md +++ b/.claude/skills/create-workflow.md @@ -120,10 +120,31 @@ The 4 pages call these methods respectively: - `self.ui.input_TOPP("ToolName", custom_defaults={})` β€” auto-generated TOPP parameter UI - `self.ui.input_python("script_name")` β€” auto-generated Python tool parameter UI - `self.ui.input_widget(key, default, label)` β€” single custom widget +- `select_input_file`, `input_TOPP` and `input_widget` accept `reactive=True` to rerun `configure()` when the widget changes (for conditional UI β€” see below) ### Logging - `self.logger.log("message")` β€” log progress during execution +## Conditional UI (reactive) + +Parameter widgets are isolated in an `st.fragment` by default, so a change reruns only that +widget and can't show/hide other widgets. Pass `reactive=True` to render the widget in the +parent scope instead β€” a change then reruns `configure()`. Read the live value from +`st.session_state` (not `self.params`, which is stale within the rerun) using +`self.parameter_manager.param_prefix` for custom-widget keys or `topp_param_prefix` for TOPP +keys of the form `":1:"`. + +```python +@st.fragment +def configure(self) -> None: + pm = self.parameter_manager + # changing the tool's `type` selectbox reruns configure() + self.ui.input_TOPP("IsobaricAnalyzer", reactive=True) + iso_type = st.session_state.get(f"{pm.topp_param_prefix}IsobaricAnalyzer:1:type", "") + if iso_type.startswith("tmt"): + self.ui.input_widget("tmt-channels", 10, "TMT channels", widget_type="number") +``` + ## Reference Files - Example workflow: `src/Workflow.py` @@ -144,6 +165,7 @@ The 4 pages call these methods respectively: - [ ] `__init__` calls `super().__init__("Name", st.session_state["workspace"])` - [ ] `upload()`, `configure()`, `execution()`, `results()` implemented - [ ] `@st.fragment` on `configure()` and `results()` +- [ ] `reactive=True` on any widget whose value controls other widgets' visibility - [ ] 4 content pages created in `content/` - [ ] All 4 pages registered as a group in `app.py` - [ ] Default parameters added to `default-parameters.json` if needed diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7dfeb75..3b4055b 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -38,7 +38,10 @@ jobs: kubectl kustomize k8s/overlays/prod/ | \ kubeconform -summary -strict -kubernetes-version 1.28.0 -skip IngressRoute - build: + build-amd64: + # amd64 path. Produces per-arch tags `--amd64`; the + # multi-arch manifest under `-` (and `latest`) is stitched + # together in `create-manifest` once the sibling `build-arm64` succeeds. needs: lint-manifests runs-on: ubuntu-latest permissions: @@ -50,6 +53,8 @@ jobs: include: - variant: full dockerfile: Dockerfile + - variant: simple + dockerfile: Dockerfile_simple steps: - uses: actions/checkout@v4 @@ -73,60 +78,491 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | - type=ref,event=branch,suffix=-${{ matrix.variant }} - type=ref,event=tag,suffix=-${{ matrix.variant }} - type=sha,prefix=,suffix=-${{ matrix.variant }} - type=raw,value=latest,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + type=ref,event=branch,suffix=-${{ matrix.variant }}-amd64 + type=ref,event=tag,suffix=-${{ matrix.variant }}-amd64 + type=sha,prefix=,suffix=-${{ matrix.variant }}-amd64 + type=raw,value=latest-amd64,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Build and conditionally push uses: docker/build-push-action@v5 with: context: . file: ${{ matrix.dockerfile }} + platforms: linux/amd64 load: true push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }} - cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2},mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} + # provenance/attestations turn the pushed tag into a manifest list, + # which the create-manifest job's `docker manifest create` then + # refuses ("is a manifest list"). Keep the push as a single-platform + # image manifest β€” same as the build-arm64 job. + provenance: false + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }}-amd64 + cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-amd64,mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} build-args: | GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - - name: Retag for kind (stable local tag) + - name: Retag for kind (image name the kustomize overlay points at) run: | - # load:true above loaded all meta-action tags into local docker. - # Retag the first one to the stable name the kustomize overlay expects. + # The prod overlay sets `newName: ghcr.io/openms/streamlit-template`, + # `newTag: main-full`. The rendered manifests reference that exact + # ref, so we need it loaded into kind under that name. Tag invariant + # across branches/variants so the test always works. FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) - docker tag "$FIRST_TAG" openms-streamlit:test + docker tag "$FIRST_TAG" ghcr.io/openms/streamlit-template:main-full - name: Save image as tar - run: docker save openms-streamlit:test -o /tmp/image.tar + run: docker save ghcr.io/openms/streamlit-template:main-full -o /tmp/image.tar - name: Upload image artifact uses: actions/upload-artifact@v4 with: - name: openms-streamlit-${{ matrix.variant }}-image + name: openms-streamlit-${{ matrix.variant }}-amd64-image path: /tmp/image.tar retention-days: 1 - test-nginx: - needs: build + build-arm64: + # arm64 path. Runs on a native ARM64 runner (no QEMU). Produces per-arch + # tags `--arm64`; gets merged into the multi-arch manifest + # under `-` by the `create-manifest` job below. The build + # uses a separate `Dockerfile.arm` / `Dockerfile_simple.arm` that swaps + # the miniforge installer to aarch64 and (for the full variant) guards + # the THIRDPARTY/Linux/aarch64 copy. The built image is also uploaded as + # an artifact so the apptainer / nginx / traefik integration jobs can + # exercise the ARM image on a native ARM runner (matrix arch=arm64). + needs: lint-manifests + runs-on: ubuntu-24.04-arm + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - variant: full + dockerfile: Dockerfile.arm + - variant: simple + dockerfile: Dockerfile_simple.arm + steps: + - name: Free disk space + # OpenMS source build needs ~25 GB of scratch space; the ARM runner + # image is tighter than the AMD one out of the box. Mirrors what + # FLASHApp's publish-docker-images.yml does at the top of its ARM job. + run: | + # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl + # cache binaries there and fail if the directory is missing. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true + sudo apt-get clean + df -h + + - uses: actions/checkout@v4 + + - name: Compute lowercase image name (OCI refs must be lowercase) + run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch,suffix=-${{ matrix.variant }}-arm64 + type=ref,event=tag,suffix=-${{ matrix.variant }}-arm64 + type=sha,prefix=,suffix=-${{ matrix.variant }}-arm64 + type=raw,value=latest-arm64,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Build and conditionally push + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/arm64 + load: true + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }}-arm64 + cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-arm64,mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} + provenance: false + build-args: | + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + + - name: Retag for kind (image name the kustomize overlay points at) + run: | + # The prod overlay sets `newName: ghcr.io/openms/streamlit-template`, + # `newTag: main-full`. The rendered manifests reference that exact + # ref, so we need it loaded into kind under that name. Tag invariant + # across branches/variants so the test always works. + FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) + docker tag "$FIRST_TAG" ghcr.io/openms/streamlit-template:main-full + + - name: Save image as tar + run: docker save ghcr.io/openms/streamlit-template:main-full -o /tmp/image.tar + + - name: Upload image artifact + uses: actions/upload-artifact@v4 + with: + name: openms-streamlit-${{ matrix.variant }}-arm64-image + path: /tmp/image.tar + retention-days: 1 + + create-manifest: + # Stitch the per-arch tags into multi-arch manifest lists. The manifest + # tags reuse the OLD scheme (`-`, `latest`) so existing + # consumers (k8s overlays, docker-compose users, `docker pull` callers) + # keep working transparently β€” docker now auto-selects the right arch + # on pull. PRs don't push per-arch tags, so there's nothing to merge. + needs: [build-amd64, build-arm64] + if: github.event_name != 'pull_request' runs-on: ubuntu-latest + permissions: + contents: read + packages: write strategy: fail-fast: false matrix: - variant: [full] + variant: [full, simple] + steps: + - name: Compute lowercase image name + run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute manifest tags + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # NB: no -amd64/-arm64 suffix here. These are the multi-arch + # manifest names; they must match the pre-arm64 tag scheme so + # `:main-full`, `:v1.0.0-full`, `:latest` continue to resolve. + tags: | + type=ref,event=branch,suffix=-${{ matrix.variant }} + type=ref,event=tag,suffix=-${{ matrix.variant }} + type=sha,prefix=,suffix=-${{ matrix.variant }} + type=raw,value=latest,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Create and push multi-arch manifests + # Iterate over manifest tags (newline-separated from metadata-action) + # and merge the matching `-amd64` / `-arm64` per-arch tags into each. + # `--amend` makes the step idempotent across workflow_dispatch reruns. + # `docker manifest push` accepts only one ref per invocation, hence + # the loop. + run: | + set -euo pipefail + while IFS= read -r manifest_tag; do + [ -z "$manifest_tag" ] && continue + amd_tag="${manifest_tag}-amd64" + arm_tag="${manifest_tag}-arm64" + echo "Creating manifest ${manifest_tag} from:" + echo " amd: ${amd_tag}" + echo " arm: ${arm_tag}" + docker manifest create "$manifest_tag" \ + --amend "$amd_tag" \ + --amend "$arm_tag" + docker manifest push "$manifest_tag" + done <<< "${{ steps.meta.outputs.tags }}" + + test-apptainer: + # Apptainer/Singularity is the dominant container runtime on HPC clusters. + # It mounts the root filesystem read-only and runs as the host user's UID + # (not root inside the image). The entrypoint must tolerate both: this job + # exercises that contract by running the built image under apptainer and + # waiting for the streamlit /_stcore/health endpoint to come up. + # + # amd64 only: upstream apptainer does NOT publish arm64 .deb assets + # (https://github.com/apptainer/apptainer/releases β€” every release lists + # only `apptainer__amd64.deb`), so eWaterCycle/setup-apptainer fails + # on ubuntu-24.04-arm with "sudo exit code 100" when its + # `apt-get install ./apptainer_*.deb` resolves a non-existent package. + # Building apptainer from source on the arm runner would add ~15 min and + # significant maintenance surface for limited value (HPC SIF consumers + # remain amd64). Re-evaluate if upstream starts publishing arm64 builds. + needs: build-amd64 + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + variant: [full, simple] steps: - uses: actions/checkout@v4 + - name: Free disk space + # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind + # node image plus loading the OCI tar into both docker and kind can + # exhaust it. The arm runner is even tighter. Same incantation as + # `build-arm64`'s "Free disk space" step. + run: | + # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl + # cache binaries there and fail if the directory is missing. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true + sudo apt-get clean + df -h + - name: Download image artifact uses: actions/download-artifact@v4 with: - name: openms-streamlit-${{ matrix.variant }}-image + name: openms-streamlit-${{ matrix.variant }}-amd64-image path: /tmp - - name: Load image into local docker - run: docker load -i /tmp/image.tar + - name: Install apptainer + uses: eWaterCycle/setup-apptainer@v2 + with: + apptainer-version: 1.3.4 + + - name: Build SIF from docker-archive + run: | + sudo apptainer build /tmp/openms.sif docker-archive:///tmp/image.tar + sudo chmod a+r /tmp/openms.sif + + - name: Prepare host bind dirs (mountpoint contract) + run: | + # Host paths we'll bind into the SIF. Asserting writability through + # singularity's bind machinery requires that the destination paths + # exist as real directories in the squashfs (otherwise singularity + # silently degrades the bind to read-only via underlay). + mkdir -p /tmp/host-workspaces /tmp/host-mounted-data + echo "from-host-pretest" > /tmp/host-mounted-data/sentinel.txt + + - name: Start apptainer instance (read-only root, host UID, with binds) + run: | + # Default apptainer semantics: read-only root, no --writable-tmpfs. + # This matches how users on HPC clusters run the SIF. + # Use `instance run` (apptainer 1.1+), not `instance start`: the SIF + # was built from docker-archive, which populates %runscript with the + # Docker ENTRYPOINT but leaves %startscript as the default no-op + # `exec "$@"`. `instance start` would launch an empty instance and + # streamlit would never bind 8501. + apptainer instance run \ + --bind /tmp/host-workspaces:/workspaces-streamlit-template:rw \ + --bind /tmp/host-mounted-data:/mounted-data:ro \ + /tmp/openms.sif openms-test + apptainer instance list + # Record where this run's logs will land so subsequent steps can tail + # them deterministically (path depends on hostname/user). + LOG_DIR=$(find "$HOME/.apptainer/instances/logs" -type d -name "$(whoami)" 2>/dev/null | head -n 1) + echo "APPTAINER_LOG_DIR=${LOG_DIR}" >> "$GITHUB_ENV" + ls -la "$LOG_DIR" || true + + - name: Wait for streamlit /_stcore/health + run: | + # Tail the entrypoint's stdout/stderr alongside the health probe so + # any startup failure surfaces directly in the CI log (the dedicated + # "Dump entrypoint logs on failure" step is post-mortem only and + # easy to miss in the GH Actions UI). + OUT="${APPTAINER_LOG_DIR}/openms-test.out" + ERR="${APPTAINER_LOG_DIR}/openms-test.err" + for i in $(seq 1 90); do + if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8501/_stcore/health; then + echo "Streamlit is ready after $i attempts" + exit 0 + fi + if [ $((i % 5)) -eq 0 ]; then + echo "--- attempt $i: instance log tail ---" + tail -n 20 "$OUT" 2>/dev/null || echo "(no $OUT yet)" + tail -n 10 "$ERR" 2>/dev/null || echo "(no $ERR yet)" + apptainer instance list || true + fi + sleep 2 + done + echo "TIMED OUT waiting for streamlit health endpoint" + echo "--- full entrypoint stdout ---" + cat "$OUT" 2>/dev/null || echo "(missing)" + echo "--- full entrypoint stderr ---" + cat "$ERR" 2>/dev/null || echo "(missing)" + exit 1 + + - name: Verify health endpoint returns 200 + run: curl -fsS http://127.0.0.1:8501/_stcore/health + + - name: Verify Redis is reachable inside container (full variant) + if: matrix.variant == 'full' + run: | + # In apptainer mode the entrypoint uses a unix socket (TCP 6379 on + # localhost is the host's, since net namespace is shared). The + # entrypoint writes the resolved URL to /tmp/openms-redis-url for + # out-of-band discovery, since `apptainer exec` spawns a fresh + # shell that doesn't inherit the daemon's exported env. + URL=$(apptainer exec instance://openms-test cat /tmp/openms-redis-url 2>/dev/null || true) + case "$URL" in + unix://*) + SOCK="${URL#unix://}" + echo "Redis URL is unix socket: $SOCK" + apptainer exec instance://openms-test redis-cli -s "$SOCK" ping | grep -i pong + ;; + *) + echo "Redis URL is TCP (or unset): ${URL:-default}" + apptainer exec instance://openms-test redis-cli ping | grep -i pong + ;; + esac + + - name: Verify bind mount is writable (workspaces) and readable (data) + run: | + # The whole point of pre-creating /workspaces-streamlit-template + # and /mounted-data in the image: singularity now has a real + # attach point and `:rw` actually sticks. Without the mkdir, + # `apptainer exec ... touch` here would fail with EROFS. + apptainer exec instance://openms-test sh -c \ + 'echo from-container > /workspaces-streamlit-template/probe.txt' + test -f /tmp/host-workspaces/probe.txt + grep -q from-container /tmp/host-workspaces/probe.txt + # Read-only data mount should also be visible inside the container. + apptainer exec instance://openms-test grep -q from-host-pretest /mounted-data/sentinel.txt + # The mounted-drive browser uses os.path.ismount() to gate + # rendering (existence is no longer enough now that the image + # pre-creates the dir). Assert the kernel reports both paths as + # real mount points so the detection function returns truthy. + apptainer exec instance://openms-test python3 -c " + import os, sys + for p in ('/mounted-data', '/workspaces-streamlit-template'): + assert os.path.ismount(p), f'{p} not reported as mount point' + print(f'ismount({p}) = True') + " + + - name: Dump entrypoint logs on failure + if: failure() + run: | + echo "--- apptainer instance list ---" + apptainer instance list || true + echo "--- apptainer instance logs ---" + find "$HOME/.apptainer" \( -name '*.out' -o -name '*.err' \) 2>/dev/null \ + | while read -r f; do echo "=== $f ==="; cat "$f"; done || true + + - name: Stop apptainer instance + if: always() + run: apptainer instance stop openms-test || true + + - name: Upload validated SIF artifact (push events only) + if: success() && github.event_name != 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: openms-streamlit-${{ matrix.variant }}-sif + path: /tmp/openms.sif + retention-days: 1 + if-no-files-found: error + + publish-apptainer: + # Publish the validated SIF (already health-checked above) to GHCR as an + # OCI artifact via ORAS, in a sibling package: ghcr.io///sif. + # Keeping it separate from the docker image package keeps tag lists clean + # and lets HPC users `apptainer pull oras://...` without the 5-15 min + # on-the-fly OCI->SIF conversion the docker:// path requires. + needs: test-apptainer + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + variant: [full, simple] + steps: + - name: Download validated SIF artifact + uses: actions/download-artifact@v4 + with: + name: openms-streamlit-${{ matrix.variant }}-sif + path: /tmp + + - name: Install apptainer + uses: eWaterCycle/setup-apptainer@v2 + with: + apptainer-version: 1.3.4 + + - name: Compute SIF tags + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/sif + tags: | + type=ref,event=branch,suffix=-${{ matrix.variant }} + type=ref,event=tag,suffix=-${{ matrix.variant }} + type=sha,prefix=,suffix=-${{ matrix.variant }} + type=raw,value=latest,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Log in to GHCR for ORAS push + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # apptainer reads its auth from ~/.apptainer/remote.yaml, NOT from + # ~/.docker/config.json β€” so docker/login-action won't work here. + # Login and push must both run as the runner user (no sudo) so they + # share the same $HOME and therefore the same auth file. + echo "$GHCR_TOKEN" | apptainer registry login \ + --username "${{ github.actor }}" \ + --password-stdin \ + oras://ghcr.io + + - name: Push SIF to each computed tag + run: | + # `apptainer push` accepts ONE destination per invocation; iterate + # over the newline-separated tag list from docker/metadata-action. + # tr lowercase is belt-and-braces β€” metadata-action already + # lowercases, but GHCR is strict about case in OCI refs. + set -euo pipefail + while IFS= read -r tag; do + [ -z "$tag" ] && continue + tag_lc="$(echo "$tag" | tr '[:upper:]' '[:lower:]')" + echo "Pushing SIF to oras://${tag_lc}" + apptainer push /tmp/openms.sif "oras://${tag_lc}" + done <<< "${{ steps.meta.outputs.tags }}" + + test-nginx: + needs: [build-amd64, build-arm64] + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - variant: full + arch: amd64 + runner: ubuntu-latest + - variant: full + arch: arm64 + runner: ubuntu-24.04-arm + - variant: simple + arch: amd64 + runner: ubuntu-latest + - variant: simple + arch: arm64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v4 + + - name: Free disk space + # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind + # node image plus loading the OCI tar into both docker and kind can + # exhaust it. The arm runner is even tighter. Same incantation as + # `build-arm64`'s "Free disk space" step. + run: | + # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl + # cache binaries there and fail if the directory is missing. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true + sudo apt-get clean + df -h + + - name: Download image artifact + uses: actions/download-artifact@v4 + with: + name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image + path: /tmp - name: Create kind cluster uses: helm/kind-action@v1 @@ -135,7 +571,13 @@ jobs: config: .github/kind-config.yaml - name: Load image into kind cluster - run: kind load docker-image openms-streamlit:test --name test-cluster + # Use `kind load image-archive` (not docker-image) so we never store + # the image in host docker. Saves ~5-8 GB on /var/lib/docker. Delete + # the tar afterwards to free the same again on /tmp β€” the image is + # now in both kind nodes' containerd, which is enough. + run: | + kind load image-archive /tmp/image.tar --name test-cluster + rm -f /tmp/image.tar - name: Install nginx ingress controller run: | @@ -147,7 +589,7 @@ jobs: # Filter out Traefik IngressRoute (kind cluster uses nginx) and force imagePullPolicy=Never kubectl kustomize k8s/overlays/prod/ | \ yq 'select(.kind != "IngressRoute")' | \ - sed 's|imagePullPolicy: IfNotPresent|imagePullPolicy: Never|g' | \ + sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' | \ sed 's|storageClassName: cinder-csi|storageClassName: standard|g' > /tmp/manifests.yaml for i in 1 2 3 4 5; do if kubectl apply -f /tmp/manifests.yaml; then @@ -196,25 +638,67 @@ jobs: echo "$host -> 200 OK" done + - name: Dump cluster state on failure + if: failure() + run: | + echo "=== nodes ===" + kubectl get nodes -o wide || true + echo "=== pods (all namespaces) ===" + kubectl get pods -A -o wide || true + echo "=== app pods describe ===" + kubectl describe pod -n openms -l app=${SLUG} || true + echo "=== app pod logs ===" + kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix || true + echo "=== app pod previous logs (if crashed) ===" + kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix --previous || true + echo "=== ingress ===" + kubectl get ingress -A -o wide || true + kubectl describe ingress -n openms || true + echo "=== services + endpoints ===" + kubectl get svc,endpoints -n openms || true + echo "=== ingress-nginx controller logs ===" + kubectl logs -n ingress-nginx -l app.kubernetes.io/component=controller --tail=200 || true + test-traefik: - needs: build - runs-on: ubuntu-latest + needs: [build-amd64, build-arm64] + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - variant: [full] + include: + - variant: full + arch: amd64 + runner: ubuntu-latest + - variant: full + arch: arm64 + runner: ubuntu-24.04-arm + - variant: simple + arch: amd64 + runner: ubuntu-latest + - variant: simple + arch: arm64 + runner: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 + - name: Free disk space + # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind + # node image plus loading the OCI tar into both docker and kind can + # exhaust it. The arm runner is even tighter. Same incantation as + # `build-arm64`'s "Free disk space" step. + run: | + # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl + # cache binaries there and fail if the directory is missing. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true + sudo apt-get clean + df -h + - name: Download image artifact uses: actions/download-artifact@v4 with: - name: openms-streamlit-${{ matrix.variant }}-image + name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image path: /tmp - - name: Load image into local docker - run: docker load -i /tmp/image.tar - - name: Create kind cluster uses: helm/kind-action@v1 with: @@ -222,7 +706,13 @@ jobs: config: .github/kind-config.yaml - name: Load image into kind cluster - run: kind load docker-image openms-streamlit:test --name traefik-test + # Use `kind load image-archive` (not docker-image) so we never store + # the image in host docker. Saves ~5-8 GB on /var/lib/docker. Delete + # the tar afterwards to free the same again on /tmp β€” the image is + # now in both kind nodes' containerd, which is enough. + run: | + kind load image-archive /tmp/image.tar --name traefik-test + rm -f /tmp/image.tar - name: Set up Helm uses: azure/setup-helm@v4 @@ -239,7 +729,7 @@ jobs: - name: Deploy with Kustomize (full manifests, no filter) run: | kubectl kustomize k8s/overlays/prod/ | \ - sed 's|imagePullPolicy: IfNotPresent|imagePullPolicy: Never|g' | \ + sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' | \ sed 's|storageClassName: cinder-csi|storageClassName: standard|g' > /tmp/manifests.yaml for i in 1 2 3 4 5; do if kubectl apply -f /tmp/manifests.yaml; then @@ -287,3 +777,23 @@ jobs: echo "" echo "$host -> 200 OK" done + + - name: Dump cluster state on failure + if: failure() + run: | + echo "=== nodes ===" + kubectl get nodes -o wide || true + echo "=== pods (all namespaces) ===" + kubectl get pods -A -o wide || true + echo "=== app pods describe ===" + kubectl describe pod -n openms -l app=${SLUG} || true + echo "=== app pod logs ===" + kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix || true + echo "=== app pod previous logs (if crashed) ===" + kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix --previous || true + echo "=== traefik ingressroute ===" + kubectl get ingressroute -A -o yaml || true + echo "=== services + endpoints ===" + kubectl get svc,endpoints -n openms || true + echo "=== traefik controller logs ===" + kubectl logs -n traefik -l app.kubernetes.io/name=traefik --tail=200 || true diff --git a/.github/workflows/build-windows-executable-app.yaml b/.github/workflows/build-windows-executable-app.yaml index e1af54b..99b4177 100644 --- a/.github/workflows/build-windows-executable-app.yaml +++ b/.github/workflows/build-windows-executable-app.yaml @@ -19,11 +19,11 @@ env: OPENMS_CONTRIB_VERSION: "" PYTHON_VERSION: 3.11.0 # Name of the installer - APP_NAME: quantms-web-DDA-LFQ + APP_NAME: OpenMS-StreamlitTemplateApp # Define unique GUID for UpgradeCode APP_UpgradeCode: "8d28e8c7-45dc-446c-b889-99a6aea2f1a5" # Define needed TOPP tools here - TOPP_TOOLS: "DecoyDatabase CometAdapter PercolatorAdapter IDFilter ProteomicsLFQ" + TOPP_TOOLS: "FeatureFinderMetabo FeatureLinkerUnlabeledKD SiriusExport" jobs: build-openms: @@ -76,7 +76,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ github.workspace }}/OpenMS/contrib - key: ${{ runner.os }}-contrib-${{ env.OPENMS_VERSION }} + key: ${{ runner.os }}-contrib-${{ env.OPENMS_CONTRIB_VERSION || env.OPENMS_VERSION }} - name: Load contrib build if: steps.cache-contrib-win.outputs.cache-hit != 'true' @@ -85,7 +85,7 @@ jobs: run: | cd OpenMS/contrib # Download the file using the URL fetched from GitHub - gh release download release/${{ env.OPENMS_VERSION }} -R OpenMS/contrib --pattern 'contrib_build-Windows.tar.gz' + gh release download release/${{ env.OPENMS_CONTRIB_VERSION || env.OPENMS_VERSION }} -R OpenMS/contrib --pattern 'contrib_build-Windows.tar.gz' # Extract the archive 7z x -so contrib_build-Windows.tar.gz | 7z x -si -ttar rm contrib_build-Windows.tar.gz diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d79f8e..34b5359 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: continuous-integration -on: [push] +on: [push, pull_request] jobs: test: @@ -8,8 +8,8 @@ jobs: strategy: matrix: os: [ubuntu-latest] - # Requirements file generated with python=3.12; tested with python=3.11 - python-version: ["3.11"] + # Match the python version used in the committed Dockerfile (release/3.5.0) + python-version: ["3.10"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 diff --git a/.github/workflows/ghcr-cleanup.yml b/.github/workflows/ghcr-cleanup.yml index 6228b62..00aca96 100644 --- a/.github/workflows/ghcr-cleanup.yml +++ b/.github/workflows/ghcr-cleanup.yml @@ -51,3 +51,29 @@ jobs: tag-selection: untagged cut-off: 7d dry-run: ${{ github.event.inputs.dry-run || 'false' }} + + cleanup-sif-images: + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: Delete old commit-tagged SIFs (keep semver + main + latest) + uses: snok/container-retention-policy@v3.0.1 + with: + account: ${{ github.repository_owner }} + token: ${{ secrets.GITHUB_TOKEN }} + image-names: ${{ github.event.repository.name }}/sif + image-tags: "!v*-full !v*-simple !main-full !main-simple !latest" + tag-selection: tagged + cut-off: 30d + dry-run: ${{ github.event.inputs.dry-run || 'false' }} + + - name: Delete untagged SIF manifests + uses: snok/container-retention-policy@v3.0.1 + with: + account: ${{ github.repository_owner }} + token: ${{ secrets.GITHUB_TOKEN }} + image-names: ${{ github.event.repository.name }}/sif + tag-selection: untagged + cut-off: 7d + dry-run: ${{ github.event.inputs.dry-run || 'false' }} diff --git a/.github/workflows/test-win-exe-w-embed-py.yaml b/.github/workflows/test-win-exe-w-embed-py.yaml index deec56d..b543d0e 100644 --- a/.github/workflows/test-win-exe-w-embed-py.yaml +++ b/.github/workflows/test-win-exe-w-embed-py.yaml @@ -1,5 +1,5 @@ name: Test streamlit executable for Windows with embeddable python -on: +on: push: branches: [ "main" ] workflow_dispatch: @@ -46,7 +46,7 @@ jobs: - name: Create .bat file run: | echo " start /min .\python-${{ env.PYTHON_VERSION }}\python -m streamlit run app.py local" > ${{ env.APP_NAME }}.bat - + - name: Create All-in-one executable folder run: | mkdir streamlit_exe @@ -86,20 +86,20 @@ jobs: - Join our Discord server for support and community discussions: https://discord.com/invite/4TAGhqJ7s5 - Contribute or stay updated with the latest OpenMS web app developments on GitHub: https://github.com/OpenMS/streamlit-template - Visit our website for more information: https://openms.de/ - + Thank you for using ${{ env.APP_NAME }}! EOF - + - name: Install WiX Toolset run: | curl -LO https://github.com/wixtoolset/wix3/releases/download/wix3111rtm/wix311-binaries.zip unzip wix311-binaries.zip -d wix rm wix311-binaries.zip - + - name: Build .wxs for streamlit_exe folder run: | ./wix/heat.exe dir streamlit_exe -gg -sfrag -sreg -srd -template component -cg StreamlitExeFiles -dr AppSubFolder -out streamlit_exe_files.wxs - + - name: Generate VBScript file shell: bash run: | @@ -115,7 +115,7 @@ jobs: cp assets/openms_license.rtf SourceDir # Logo of app cp assets/openms.ico SourceDir - + - name: Generate WiX XML file shell: bash run: | @@ -125,13 +125,13 @@ jobs: - + - + - + @@ -141,7 +141,7 @@ jobs: - + @@ -149,30 +149,30 @@ jobs: - + - + - + - + - + - + @@ -180,13 +180,13 @@ jobs: - + - - + + @@ -196,7 +196,7 @@ jobs: - name: Build .wixobj file with candle.exe run: | ./wix/candle.exe streamlit_exe.wxs streamlit_exe_files.wxs - + - name: Link .wixobj file into .msi with light.exe run: | ./wix/light.exe -ext WixUIExtension -sice:ICE60 -o ${{ env.APP_NAME }}.msi streamlit_exe_files.wixobj streamlit_exe.wixobj @@ -206,4 +206,4 @@ jobs: with: name: OpenMS-App-Test path: | - ${{ env.APP_NAME }}.msi + ${{ env.APP_NAME }}.msi \ No newline at end of file diff --git a/.github/workflows/test-win-exe-w-pyinstaller.yaml b/.github/workflows/test-win-exe-w-pyinstaller.yaml index 15e7ef4..94c91ae 100644 --- a/.github/workflows/test-win-exe-w-pyinstaller.yaml +++ b/.github/workflows/test-win-exe-w-pyinstaller.yaml @@ -19,11 +19,10 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Setup virtual environment - shell: cmd + shell: cmd run: | python -m venv myenv - call myenv\Scripts\activate.bat - pip install cython numpy + call myenv\Scripts\activate.bat pip install -r requirements.txt pip install pyinstaller diff --git a/.github/workflows/workflow-tests.yml b/.github/workflows/workflow-tests.yml new file mode 100644 index 0000000..92b0b99 --- /dev/null +++ b/.github/workflows/workflow-tests.yml @@ -0,0 +1,28 @@ +name: Test workflow functions + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest + - name: Running test cases + run: | + pytest test.py + - name: Running GUI tests + run: | + pytest test_gui.py diff --git a/.gitignore b/.gitignore index 227f773..1525129 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ gdpr_consent/node_modules/ *~ .streamlit/secrets.toml docs/superpowers/ +.venv/ \ No newline at end of file diff --git a/.streamlit/config.toml b/.streamlit/config.toml index f54abde..b7f3cf6 100644 --- a/.streamlit/config.toml +++ b/.streamlit/config.toml @@ -10,7 +10,8 @@ developmentMode = false files = ["/app/admin-secrets/secrets.toml", "~/.streamlit/secrets.toml", ".streamlit/secrets.toml"] [server] -maxUploadSize = 1000 #MB +address = "0.0.0.0" +maxUploadSize = 200 #MB port = 8501 # should be same as configured in deployment repo diff --git a/CLAUDE.md b/CLAUDE.md index 948210c..e0e0a22 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,94 +1,173 @@ -# CLAUDE.md +# OpenMS Streamlit WebApp Template -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +## What This Is -## Project Overview +**This is the standard framework for building web applications for mass spectrometry (MS) data analysis**, used across the OpenMS ecosystem for proteomics and metabolomics research. When a researcher or developer needs a web-based tool for MS data processing, visualization, or analysis β€” whether for label-free quantification, untargeted metabolomics, top-down proteomics, or any other MS workflow β€” this template is how it gets built. -OpenMS Streamlit Template is a web application framework for building mass spectrometry (MS) analysis workflows using OpenMS/pyOpenMS. It supports both simple pyOpenMS workflows and complex multi-tool pipelines using OpenMS TOPP (The OpenMS Proteomics Pipeline) tools. +The template wraps **OpenMS/pyOpenMS** (the leading open-source C++/Python library for computational mass spectrometry) and its **TOPP tools** (a suite of ~200 command-line tools for MS data processing pipelines) into interactive Streamlit web applications. -## Common Commands +### Production Apps Built From This Template -```bash -# Run the app locally -streamlit run app.py +- **OpenMS/quantms-web** β€” quantitative proteomics (DDA-LFQ, DDA-ISO, DIA-LFQ quantification) +- **OpenMS/umetaflow** β€” untargeted metabolomics (feature detection, alignment, annotation, GNPS molecular networking) +- **OpenMS/FLASHApp** β€” top-down proteomics (FLASHDeconv deconvolution result visualization) -# Run tests -python -m pytest test_gui.py tests/ +### Mass Spectrometry Domain Context -# Build and run with Docker (includes OpenMS TOPP tools) -docker-compose up -d --build +- **Input data** is typically mzML (raw MS spectra), featureXML (detected features), consensusXML (linked features across samples), idXML (peptide/protein identifications), traML (targeted transitions) +- **Typical workflows chain TOPP tools**: e.g., `FeatureFinderMetabo` (detect LC-MS features) β†’ `FeatureLinkerUnlabeledKD` (align features across runs) β†’ custom Python post-processing +- **Proteomics** focuses on peptide/protein identification and quantification (tools like `MSGFPlusAdapter`, `FidoAdapter`, `ProteinQuantifier`) +- **Metabolomics** focuses on feature detection, annotation, and statistical analysis (tools like `FeatureFinderMetabo`, `MetaboliteAdductDecharger`, `SiriusAdapter`) +- **pyOpenMS** provides Python bindings for programmatic MS data access β€” reading mzML files, manipulating spectra/chromatograms, computing molecular properties, etc. +- **MS-specific visualizations**: mass spectra (m/z vs intensity), chromatograms (RT vs intensity), peak maps (RT vs m/z 2D heatmaps), isotope patterns, fragment ion annotations, volcano plots for differential expression + +## Architecture -# Clean up old workspaces (removes workspaces older than 7 days) -python clean-up-workspaces.py +``` +app.py # Entry point β€” registers pages via st.Page() in a dict +settings.json # App config: name, version, deployment mode, threading +default-parameters.json # Default workspace parameters (tracked via widget keys) +presets.json # Parameter presets for TOPP workflows +content/ # Streamlit pages (one .py per page) +src/ + common/common.py # Utilities: page_setup(), save_params(), show_fig(), show_table() + Workflow.py # Example WorkflowManager subclass (TOPP workflow) + workflow/ + WorkflowManager.py # Base class: upload/configure/execution/results pattern + StreamlitUI.py # Widget library: upload_widget, input_TOPP, input_python, etc. + ParameterManager.py # JSON parameter persistence + TOPP .ini generation + CommandExecutor.py # Runs TOPP tools and Python scripts as subprocesses + FileManager.py # Workspace file organization + Logger.py # Structured workflow logging + QueueManager.py # Redis queue for online deployments + python-tools/ # Custom Python analysis scripts (with DEFAULTS dicts) +Dockerfile # Full build: OpenMS + TOPP tools + pyOpenMS +Dockerfile_simple # Lightweight: pyOpenMS only +docker-compose.yml # Deployment config ``` -Note: Local runs have limited functionality. Features requiring OpenMS TOPP tools only work out of the box with Docker or when OpenMS Command Line Tools are installed separately. +## Key Patterns -## Architecture +### Pages -### Core Framework (`src/workflow/`) +Every page starts with `page_setup()` which handles workspace initialization, sidebar rendering, and parameter loading: -The workflow system is built around `WorkflowManager` as the base class with these components: +```python +from src.common.common import page_setup, save_params +params = page_setup() +``` -- **WorkflowManager**: Base class that orchestrates file management, parameters, command execution, and UI. Custom workflows inherit from this and override `upload()`, `configure()`, `execution()`, and `results()` methods. -- **FileManager**: Handles input/output file organization in `workflow_dir/input-files/{key}/` and `workflow_dir/results/` -- **ParameterManager**: Manages TOPP tool parameters (XML .ini files) and JSON parameters -- **CommandExecutor**: Runs external commands (TOPP tools) with threading for parallelization -- **StreamlitUI**: Provides Streamlit widgets including `upload_widget()` and `input_TOPP()` for TOPP parameter UIs -- **Logger**: Multi-level logging (minimal, commands, all) to `workflow_dir/logs/` +Pages are registered in `app.py` under named sections: -### Page Structure +```python +pages = { + "Section Name": [ + st.Page(Path("content", "my_page.py"), title="My Page", icon="πŸ”¬"), + ], +} +``` -- **Entry point**: `app.py` defines multi-page navigation using `st.navigation()` -- **Pages**: Each file in `content/` is a page that calls `page_setup()` from `src/common/common.py`, then instantiates a workflow class -- **Utility pages** (`content/`): digest.py, fragmentation.py, isotope_pattern_generator.py, peptide_mz_calculator.py provide standalone analysis tools +### Parameters -### Workflow Data Flow +Parameters are tracked via widget keys that match entries in `default-parameters.json`. The `save_params(params)` call at the end of a page persists any widget state changes: + +```python +params = page_setup() +st.number_input("X", value=params["my-param"], key="my-param") +save_params(params) +``` -1. `page_setup()` initializes workspace and loads parameters from `workspace/params.json` -2. Workflow class (e.g., `WorkflowTest`) inherits from `WorkflowManager` -3. Each page calls the appropriate method: `show_file_upload_section()`, `show_parameter_section()`, `show_execution_section()`, `show_results_section()` -4. Workflow execution runs in a multiprocessing.Process to avoid blocking Streamlit UI updates +### TOPP Workflows (WorkflowManager) -### Key Patterns +Complex workflows subclass `WorkflowManager` and implement 4 methods: +- `upload()` β€” file upload widgets via `self.ui.upload_widget()` +- `configure()` β€” TOPP params via `self.ui.input_TOPP()`, Python tool params via `self.ui.input_python()` +- `execution()` β€” run tools via `self.executor.run_topp()` and `self.executor.run_python()` +- `results()` β€” display outputs -- **Workspace isolation**: Each user session gets a unique workspace directory for files and parameters -- **Streamlit fragments**: Use `@st.fragment` decorator for interactive UI updates without full page reloads -- **TOPP tool execution**: `executor.run_topp("ToolName", {inputs/outputs}, {extra_params})` handles parameter files and command construction +Each workflow gets 4 content pages (upload, configure, run, results) that call `wf.show_*_section()`. -## Configuration Files +Decorate `configure()` and `results()` with `@st.fragment` for partial reruns. -- `settings.json`: App name, version, analytics, workspace settings -- `default-parameters.json`: Workflow default parameters -- `.streamlit/config.toml`: Streamlit server config (port 8501, 1000MB upload limit) +For conditional UI (a widget that shows/hides other widgets), pass `reactive=True` to `input_widget`, `select_input_file`, or `input_TOPP` so a change reruns the parent `configure()` instead of only its isolated fragment. Read the changed value from `st.session_state` (not `self.params`, which is stale within the rerun) via `parameter_manager.param_prefix` for custom-widget keys or `topp_param_prefix` for TOPP keys (`":1:"`). -## Creating New Workflows +### Python Tools -Inherit from `WorkflowManager` and implement the four core methods: +Custom scripts in `src/python-tools/` define a `DEFAULTS` list for auto-generated UI: ```python -from src.workflow.WorkflowManager import WorkflowManager +DEFAULTS = [ + {"key": "in", "value": [], "hide": True}, + {"key": "my-param", "value": 5, "name": "My Parameter", "help": "Description", + "min": 1, "max": 100, "step_size": 1, "widget_type": "slider"}, +] +``` -class MyWorkflow(WorkflowManager): - def __init__(self): - super().__init__("My Workflow", st.session_state["workspace"]) +### Presets - def upload(self): - self.ui.upload_widget(key="input-files", name="Input", file_types="mzML", fallback=[...]) +Parameter presets in `presets.json` map workflow names (lowercase, hyphens) to named parameter sets: + +```json +{ + "workflow-name": { + "Preset Name": { + "_description": "Tooltip text", + "TOPPToolName": {"algorithm:section:param": value}, + "_general": {"custom-key": value} + } + } +} +``` - def configure(self): - self.ui.input_TOPP("ToolName", custom_defaults={...}, include_parameters=[...]) +## Visualization Libraries - def execution(self): - self.executor.run_topp("ToolName", {"in": [...], "out": [...]}, {...}) +Two libraries are commonly used in template-based apps for MS data visualization: + +### pyopenms-viz + +Pandas DataFrame extension for MS visualization. Use the plotly backend in Streamlit: + +```python +import pyopenms_viz +df.plot.ms_spectrum(backend="plotly") # mass spectrum (m/z vs intensity) +df.plot.peak_map(backend="plotly") # 2D peak map (RT vs m/z heatmap) +df.plot.chromatogram(backend="plotly") # chromatogram (RT vs intensity) +df.plot.mobilogram(backend="plotly") # ion mobility trace +``` + +Best for: publication-quality static/interactive plots, small-medium datasets, standard MS plot types. + +### OpenMS-Insight (t0mdavid-m/openms-insight) + +Vue.js-backed interactive Streamlit components for large MS datasets: + +- `Table` β€” server-side pagination with Tabulator.js +- `LinePlot` β€” stick-style mass spectra via Plotly +- `Heatmap` β€” 2D scatter handling millions of points +- `VolcanoPlot` β€” differential expression visualization +- `SequenceView` β€” peptide sequence with fragment ion matching + +Components support cross-linking via shared identifiers. Best for: large datasets (millions of points), cross-component interactivity, server-side pagination. + +## Commands + +```bash +# Run locally +pip install -r requirements.txt +streamlit run app.py + +# Run tests +python -m pytest tests/ - def results(self): - st.dataframe(pd.read_csv(...)) +# Docker +docker-compose up --build ``` -## Key Dependencies +## Conventions -- **pyOpenMS 3.5.0+**: Python bindings for OpenMS -- **Streamlit 1.43.0**: Web UI framework -- **Plotly + streamlit_plotly_events**: Interactive visualizations -- **OpenMS TOPP tools**: External command-line tools (Docker or separate install required) +- Page files go in `content/`, source logic in `src/` +- Widget keys must match parameter keys in `default-parameters.json` +- Workflow names use lowercase with hyphens: "My Workflow" -> "my-workflow" +- Use `show_fig()` and `show_table()` from `src/common/common.py` for consistent display +- Use `@st.fragment` on methods that should partially rerun (configure, results) +- TOPP tool parameters use colon-separated paths: `"algorithm:section:param_name"` diff --git a/Dockerfile b/Dockerfile index 2d72377..2d1b5da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ ARG PORT=8501 # Streamlit app GitHub user name (to download artifact from). ARG GITHUB_USER=OpenMS # Streamlit app GitHub repository name (to download artifact from). -ARG GITHUB_REPO=quantms-web +ARG GITHUB_REPO=streamlit-template USER root @@ -47,6 +47,13 @@ RUN wget -q \ && rm -f Miniforge3-Linux-x86_64.sh RUN mamba --version +# Make /root traversable so the entrypoint can `source +# /root/miniforge3/bin/activate ...` when the container runs as a non-root +# user (apptainer/singularity maps the host UID into the container; the +# default ubuntu /root is 0700 which would block path traversal). +x only, +# not +r, so the directory listing remains private. +RUN chmod o+x /root + # Setup mamba environment. RUN mamba create -n streamlit-env python=3.10 RUN echo "mamba activate streamlit-env" >> ~/.bashrc @@ -78,14 +85,20 @@ RUN mkdir /openms-build WORKDIR /openms-build # Configure. -RUN /bin/bash -c "cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF -DPYOPENMS=OFF ../OpenMS" +RUN /bin/bash -c "cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF -DPYOPENMS=ON ../OpenMS -DPY_MEMLEAK_DISABLE=On" # Build TOPP tools and clean up. RUN make -j4 TOPP RUN rm -rf src doc CMakeFiles -# Install dependencies (pyopenms will be installed from pip) -COPY requirements.txt ./requirements.txt +# Build pyOpenMS wheels and install via pip. +RUN make -j4 pyopenms +WORKDIR /openms-build/pyOpenMS +RUN pip install dist/*.whl + +# Install other dependencies (excluding pyopenms) +COPY requirements.txt ./requirements.txt +RUN grep -Ev '^pyopenms([=<>!~].*)?$' requirements.txt > requirements_cleaned.txt && mv requirements_cleaned.txt requirements.txt RUN pip install -r requirements.txt WORKDIR / @@ -110,12 +123,26 @@ RUN rm -rf openms-build # Prepare and run streamlit app. FROM compile-openms AS run-app -# Install Redis server for job queue and nginx for load balancing +# Install Redis server for job queue and nginx for load balancing. +# Redis data lives under $RUNTIME_DIR at runtime (see entrypoint.sh) so no +# /var/lib/redis setup is needed - that path is not writable under Apptainer. RUN apt-get update && apt-get install -y --no-install-recommends redis-server nginx \ && rm -rf /var/lib/apt/lists/* -# Create Redis data directory -RUN mkdir -p /var/lib/redis && chown redis:redis /var/lib/redis +# Create Redis data directory. Default 0755 root-owned is enough: the docker +# entrypoint runs as root (can write regardless of mode), and the apptainer +# entrypoint relocates Redis state to /tmp/openms-runtime-* so this dir is +# never written under apptainer. +RUN mkdir -p /var/lib/redis + +# Pre-create bind-mount targets so apptainer/singularity has a real attach +# point. Docker auto-creates missing `-v` targets, but singularity uses a +# read-only underlay and silently ignores `:rw` when the target isn't a +# real directory in the SIF β€” writes then fail with EROFS even though the +# host bind path is writable. Pre-creating these directories costs one +# inode each and changes nothing in docker mode (the user's volume mount +# shadows them). +RUN mkdir -p /workspaces-streamlit-template /mounted-data # Create workdir and copy over all streamlit related files/folders. @@ -123,6 +150,7 @@ RUN mkdir -p /var/lib/redis && chown redis:redis /var/lib/redis WORKDIR /app COPY assets/ /app/assets COPY content/ /app/content +COPY docs/ /app/docs COPY example-data/ /app/example-data COPY gdpr_consent/ /app/gdpr_consent COPY hooks/ /app/hooks @@ -148,67 +176,10 @@ ENV REDIS_URL=redis://localhost:6379/0 # Set to >1 to enable nginx load balancer with multiple Streamlit instances ENV STREAMLIT_SERVER_COUNT=1 -# create entrypoint script to start cron, Redis, RQ workers, and Streamlit -RUN echo -e '#!/bin/bash\n\ -set -e\n\ -source /root/miniforge3/bin/activate streamlit-env\n\ -\n\ -# Start cron for workspace cleanup\n\ -service cron start\n\ -\n\ -# Start Redis server in background\n\ -echo "Starting Redis server..."\n\ -redis-server --daemonize yes --dir /var/lib/redis --appendonly no\n\ -\n\ -# Wait for Redis to be ready\n\ -until redis-cli ping > /dev/null 2>&1; do\n\ - echo "Waiting for Redis..."\n\ - sleep 1\n\ -done\n\ -echo "Redis is ready"\n\ -\n\ -# Start RQ worker(s) in background\n\ -WORKER_COUNT=${RQ_WORKER_COUNT:-1}\n\ -echo "Starting $WORKER_COUNT RQ worker(s)..."\n\ -for i in $(seq 1 $WORKER_COUNT); do\n\ - rq worker openms-workflows --url $REDIS_URL --name worker-$i &\n\ -done\n\ -\n\ -# Load balancer setup\n\ -SERVER_COUNT=${STREAMLIT_SERVER_COUNT:-1}\n\ -\n\ -if [ "$SERVER_COUNT" -gt 1 ]; then\n\ - echo "Starting $SERVER_COUNT Streamlit instances with nginx load balancer..."\n\ -\n\ - # Generate nginx upstream block\n\ - UPSTREAM_SERVERS=""\n\ - BASE_PORT=8510\n\ - for i in $(seq 0 $((SERVER_COUNT - 1))); do\n\ - PORT=$((BASE_PORT + i))\n\ - UPSTREAM_SERVERS="${UPSTREAM_SERVERS} server 127.0.0.1:${PORT};\\n"\n\ - done\n\ -\n\ - # Write nginx config\n\ - mkdir -p /etc/nginx\n\ - echo -e "worker_processes auto;\\npid /run/nginx.pid;\\n\\nevents {\\n worker_connections 1024;\\n}\\n\\nhttp {\\n client_max_body_size 0;\\n\\n map \\$cookie_stroute \\$route_key {\\n \\x22\\x22 \\$request_id;\\n default \\$cookie_stroute;\\n }\\n\\n upstream streamlit_backend {\\n hash \\$route_key consistent;\\n${UPSTREAM_SERVERS} }\\n\\n map \\$http_upgrade \\$connection_upgrade {\\n default upgrade;\\n \\x27\\x27 close;\\n }\\n\\n server {\\n listen 0.0.0.0:8501;\\n\\n location / {\\n proxy_pass http://streamlit_backend;\\n proxy_http_version 1.1;\\n proxy_set_header Upgrade \\$http_upgrade;\\n proxy_set_header Connection \\$connection_upgrade;\\n proxy_set_header Host \\$host;\\n proxy_set_header X-Real-IP \\$remote_addr;\\n proxy_set_header X-Forwarded-For \\$proxy_add_x_forwarded_for;\\n proxy_set_header X-Forwarded-Proto \\$scheme;\\n proxy_read_timeout 86400;\\n proxy_send_timeout 86400;\\n proxy_buffering off;\\n add_header Set-Cookie \\x22stroute=\\$route_key; Path=/; HttpOnly; SameSite=Lax\\x22 always;\\n }\\n }\\n}" > /etc/nginx/nginx.conf\n\ -\n\ - # Start Streamlit instances on internal ports\n\ - for i in $(seq 0 $((SERVER_COUNT - 1))); do\n\ - PORT=$((BASE_PORT + i))\n\ - echo "Starting Streamlit instance on port $PORT..."\n\ - streamlit run app.py --server.port $PORT --server.address 0.0.0.0 &\n\ - done\n\ -\n\ - sleep 2\n\ - echo "Starting nginx load balancer on port 8501..."\n\ - exec /usr/sbin/nginx -g "daemon off;"\n\ -else\n\ - # Single instance mode (default) - run Streamlit directly on port 8501\n\ - echo "Starting Streamlit app..."\n\ - exec streamlit run app.py --server.address 0.0.0.0\n\ -fi\n\ -' > /app/entrypoint.sh -# make the script executable +# Install the apptainer-compatible entrypoint that starts cron (when the root +# FS is writable), Redis, RQ workers, optional nginx load balancer, and the +# Streamlit server. The script falls back to /tmp paths under apptainer. +COPY docker/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh # Patch Analytics @@ -217,6 +188,11 @@ RUN mamba run -n streamlit-env python hooks/hook-analytics.py # Set Online Deployment RUN jq '.online_deployment = true' settings.json > tmp.json && mv tmp.json settings.json +# Point the in-app mounted-drive browser at the conventional bind-mount path. +# The browser only renders when this directory exists at runtime, i.e. when +# the user starts the container with `-v /host/path:/mounted-data`. +RUN jq '.local_data_dir = "/mounted-data"' settings.json > tmp.json && mv tmp.json settings.json + # Download latest OpenMS App executable as a ZIP file. # ARG declared here (not at the top) β€” otherwise the per-run token busts the cache. ARG GITHUB_TOKEN diff --git a/Dockerfile.arm b/Dockerfile.arm new file mode 100644 index 0000000..1765980 --- /dev/null +++ b/Dockerfile.arm @@ -0,0 +1,237 @@ +# This Dockerfile builds OpenMS, the TOPP tools, pyOpenMS and thidparty tools. +# It also adds a basic streamlit server that serves a pyOpenMS-based app. +# hints: +# build image and give it a name (here: streamlitapp) with: docker build -f Dockerfile.arm --no-cache -t streamlitapp:latest-arm64 --build-arg GITHUB_TOKEN= . 2>&1 | tee build.log +# check if image was build: docker image ls +# run container: docker run -p 8501:8501 streamlitappsimple:latest +# debug container after build (comment out ENTRYPOINT) and run container with interactive /bin/bash shell +# prune unused images/etc. to free disc space (e.g. might be needed on gitpod). Use with care.: docker system prune --all --force + +FROM ubuntu:22.04 AS setup-build-system +ARG OPENMS_REPO=https://github.com/OpenMS/OpenMS.git +ARG OPENMS_BRANCH=release/3.5.0 +ARG PORT=8501 +# Streamlit app GitHub user name (to download artifact from). +ARG GITHUB_USER=OpenMS +# Streamlit app GitHub repository name (to download artifact from). +ARG GITHUB_REPO=streamlit-template + +USER root + +# Install required Ubuntu packages. +RUN apt-get -y update +RUN apt-get install -y --no-install-recommends --no-install-suggests g++ autoconf automake patch libtool make git gpg wget ca-certificates curl jq libgtk2.0-dev openjdk-8-jdk cron cmake +RUN update-ca-certificates +RUN apt-get install -y --no-install-recommends --no-install-suggests libsvm-dev libeigen3-dev coinor-libcbc-dev libglpk-dev libzip-dev zlib1g-dev libxerces-c-dev libbz2-dev libomp-dev libhdf5-dev +RUN apt-get install -y --no-install-recommends --no-install-suggests libboost-date-time1.74-dev \ + libboost-iostreams1.74-dev \ + libboost-regex1.74-dev \ + libboost-math1.74-dev \ + libboost-random1.74-dev +RUN apt-get install -y --no-install-recommends --no-install-suggests qt6-base-dev libqt6svg6-dev libqt6opengl6-dev libqt6openglwidgets6 libgl-dev + +# Install Github CLI +RUN (type -p wget >/dev/null || (apt-get update && apt-get install wget -y)) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt-get update \ + && apt-get install gh -y + +# Download and install miniforge. +ENV PATH="/root/miniforge3/bin:${PATH}" +RUN wget -q \ + https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh \ + && bash Miniforge3-Linux-aarch64.sh -b \ + && rm -f Miniforge3-Linux-aarch64.sh +RUN mamba --version + +# Make /root traversable so the entrypoint can `source +# /root/miniforge3/bin/activate ...` when the container runs as a non-root +# user (apptainer/singularity maps the host UID into the container; the +# default ubuntu /root is 0700 which would block path traversal). +x only, +# not +r, so the directory listing remains private. +RUN chmod o+x /root + +# Setup mamba environment. +RUN mamba create -n streamlit-env python=3.10 +RUN echo "mamba activate streamlit-env" >> ~/.bashrc +SHELL ["/bin/bash", "--rcfile", "~/.bashrc"] +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +# Install up-to-date cmake via mamba and packages for pyOpenMS build. +RUN mamba install cmake +RUN pip install --upgrade pip && python -m pip install -U setuptools nose cython "autowrap<=0.24" pandas numpy pytest + +# Clone OpenMS branch and the associcated contrib+thirdparties+pyOpenMS-doc submodules. +RUN git clone --recursive --depth=1 -b ${OPENMS_BRANCH} --single-branch ${OPENMS_REPO} && cd /OpenMS + +# Pull Linux compatible third-party dependencies and store them in directory thirdparty. +WORKDIR /OpenMS +RUN mkdir /thirdparty && \ + git submodule update --init THIRDPARTY && \ + cp -r THIRDPARTY/All/* /thirdparty && \ + if [ -d "THIRDPARTY/Linux/aarch64" ]; then \ + cp -r THIRDPARTY/Linux/aarch64/* /thirdparty; \ + fi && \ + chmod -R +x /thirdparty +ENV PATH="/thirdparty/LuciPHOr2:/thirdparty/MSGFPlus:/thirdparty/ThermoRawFileParser:/thirdparty/Comet:/thirdparty/Percolator:/thirdparty/Sage:${PATH}" + +# Build OpenMS and pyOpenMS. +FROM setup-build-system AS compile-openms +WORKDIR / + +# Set up build directory. +RUN mkdir /openms-build +WORKDIR /openms-build + +# Configure (two-pass β€” mirrors FLASHApp.arm). +# Pass 1 runs under plain bash so cmake does NOT search /root/miniforge3 +# when resolving C++ system dependencies. On ARM the conda-forge build of +# libyaml-cpp.so.0.8 is linked against a newer libstdc++ (GLIBCXX_3.4.32, +# i.e. gcc 13+) than ubuntu:22.04's system g++ ships, so letting cmake +# pick the miniforge yaml-cpp makes TOPP linking fail with +# undefined reference to `std::ios_base_library_init()@GLIBCXX_3.4.32` +# amd64 happens to work because its conda-forge yaml-cpp build is older. +# We call the mamba-env cmake by full path so we get a version >= 3.24 +# (OpenMS 3.5's floor); ubuntu:22.04's apt cmake is 3.22 which is too old. +# CMAKE_IGNORE_PREFIX_PATH keeps cmake from auto-discovering miniforge libs +# even though the binary itself lives there. +# Pass 2 re-runs cmake inside the mamba env with PYOPENMS=ON so the Python +# bindings can find the conda-forge Python/Cython/NumPy; CMAKE_IGNORE_PREFIX_PATH +# keeps the C++ link command unchanged from pass 1. +SHELL ["/bin/bash", "-c"] +RUN /root/miniforge3/envs/streamlit-env/bin/cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DCMAKE_IGNORE_PREFIX_PATH=/root/miniforge3 -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF ../OpenMS +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] +RUN cmake -DPYOPENMS=ON -DPY_MEMLEAK_DISABLE=On -DCMAKE_IGNORE_PREFIX_PATH=/root/miniforge3 . + +# Build TOPP tools and clean up. +RUN make -j4 TOPP +# NOTE: do NOT delete CMakeFiles/ here. The two-pass cmake configure used +# above generates CMakeFiles/VerifyGlobs.cmake for the pyOpenMS targets' +# CONFIGURE_DEPENDS globs; the next `make -j4 pyopenms` runs +# `cmake_check_build_system` which fails fast if VerifyGlobs.cmake is gone: +# CMake Error: Not a file: /openms-build/CMakeFiles/VerifyGlobs.cmake +# The x86 single-pass build seems to avoid generating that file (different +# cmake codepath when PYOPENMS is set during the initial configure), which +# is why it can still `rm -rf CMakeFiles` here. CMakeFiles/ adds ~a few +# hundred MB to the intermediate layer β€” acceptable. +RUN rm -rf src doc + +# Build pyOpenMS wheels and install via pip. +RUN make -j4 pyopenms +WORKDIR /openms-build/pyOpenMS +RUN pip install dist/*.whl + +# Install other dependencies (excluding pyopenms) +COPY requirements.txt ./requirements.txt +RUN grep -Ev '^pyopenms([=<>!~].*)?$' requirements.txt > requirements_cleaned.txt && mv requirements_cleaned.txt requirements.txt +RUN pip install -r requirements.txt + +WORKDIR / +RUN mkdir openms + +# Copy TOPP tools bin directory, add to PATH. +RUN cp -r openms-build/bin /openms/bin +ENV PATH="/openms/bin/:${PATH}" + +# Copy TOPP tools bin directory, add to PATH. +RUN cp -r openms-build/lib /openms/lib +ENV LD_LIBRARY_PATH="/openms/lib/:${LD_LIBRARY_PATH}" + +# Copy share folder, add to PATH, remove source directory. +RUN cp -r OpenMS/share/OpenMS /openms/share +RUN rm -rf OpenMS +ENV OPENMS_DATA_PATH="/openms/share/" + +# Remove build directory. +RUN rm -rf openms-build + +# Prepare and run streamlit app. +FROM compile-openms AS run-app + +# Install Redis server for job queue and nginx for load balancing. +# Redis data lives under $RUNTIME_DIR at runtime (see entrypoint.sh) so no +# /var/lib/redis setup is needed - that path is not writable under Apptainer. +RUN apt-get update && apt-get install -y --no-install-recommends redis-server nginx \ + && rm -rf /var/lib/apt/lists/* + +# Create Redis data directory. Default 0755 root-owned is enough: the docker +# entrypoint runs as root (can write regardless of mode), and the apptainer +# entrypoint relocates Redis state to /tmp/openms-runtime-* so this dir is +# never written under apptainer. +RUN mkdir -p /var/lib/redis + +# Pre-create bind-mount targets so apptainer/singularity has a real attach +# point. Docker auto-creates missing `-v` targets, but singularity uses a +# read-only underlay and silently ignores `:rw` when the target isn't a +# real directory in the SIF β€” writes then fail with EROFS even though the +# host bind path is writable. Pre-creating these directories costs one +# inode each and changes nothing in docker mode (the user's volume mount +# shadows them). +RUN mkdir -p /workspaces-streamlit-template /mounted-data + +# Create workdir and copy over all streamlit related files/folders. + +# note: specifying folder with slash as suffix and repeating the folder name seems important to preserve directory structure +WORKDIR /app +COPY assets/ /app/assets +COPY content/ /app/content +COPY docs/ /app/docs +COPY example-data/ /app/example-data +COPY gdpr_consent/ /app/gdpr_consent +COPY hooks/ /app/hooks +COPY src/ /app/src +COPY utils/ /app/utils +COPY app.py /app/app.py +COPY settings.json /app/settings.json +COPY default-parameters.json /app/default-parameters.json +COPY presets.json /app/presets.json + +# For streamlit configuration +COPY .streamlit/ /app/.streamlit/ +COPY clean-up-workspaces.py /app/clean-up-workspaces.py + +# add cron job to the crontab +RUN echo "0 3 * * * /root/miniforge3/envs/streamlit-env/bin/python /app/clean-up-workspaces.py >> /app/clean-up-workspaces.log 2>&1" | crontab - + +# Set default worker count (can be overridden via environment variable) +ENV RQ_WORKER_COUNT=1 +ENV REDIS_URL=redis://localhost:6379/0 + +# Number of Streamlit server instances for load balancing (default: 1 = no load balancer) +# Set to >1 to enable nginx load balancer with multiple Streamlit instances +ENV STREAMLIT_SERVER_COUNT=1 + +# Install the apptainer-compatible entrypoint that starts cron (when the root +# FS is writable), Redis, RQ workers, optional nginx load balancer, and the +# Streamlit server. The script falls back to /tmp paths under apptainer. +COPY docker/entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh + +# Patch Analytics +RUN mamba run -n streamlit-env python hooks/hook-analytics.py + +# Set Online Deployment +RUN jq '.online_deployment = true' settings.json > tmp.json && mv tmp.json settings.json + +# Point the in-app mounted-drive browser at the conventional bind-mount path. +# The browser only renders when this directory exists at runtime, i.e. when +# the user starts the container with `-v /host/path:/mounted-data`. +RUN jq '.local_data_dir = "/mounted-data"' settings.json > tmp.json && mv tmp.json settings.json + +# Download latest OpenMS App executable as a ZIP file. +# ARG declared here (not at the top) β€” otherwise the per-run token busts the cache. +ARG GITHUB_TOKEN +RUN if [ -n "$GITHUB_TOKEN" ]; then \ + echo "GITHUB_TOKEN is set, proceeding to download the release asset..."; \ + gh release download -R ${GITHUB_USER}/${GITHUB_REPO} -p "OpenMS-App.zip" -D /app; \ + else \ + echo "GITHUB_TOKEN is not set, skipping the release asset download."; \ + fi + + +# Run app as container entrypoint. +EXPOSE $PORT +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/Dockerfile_simple b/Dockerfile_simple new file mode 100644 index 0000000..163bcfe --- /dev/null +++ b/Dockerfile_simple @@ -0,0 +1,127 @@ +# This Dockerfile creates a container with pyOpenMS +# It also adds a basic streamlit server that serves a pyOpenMS-based app. +# hints: +# build image with: docker build -f Dockerfile_simple --no-cache -t streamlitapp:latest --build-arg GITHUB_TOKEN= . 2>&1 | tee build.log +# check if image was build: docker image ls +# run container: docker run -p 8501:8501 streamlitapp:latest +# debug container after build (comment out ENTRYPOINT) and run container with interactive /bin/bash shell +# prune unused images/etc. to free disc space (e.g. might be needed on gitpod). Use with care.: docker system prune --all --force + +FROM ubuntu:22.04 AS stage1 +ARG OPENMS_REPO=https://github.com/OpenMS/OpenMS.git +ARG OPENMS_BRANCH=develop +ARG PORT=8501 +# Streamlit app GitHub user name (to download artifact from). +ARG GITHUB_USER=OpenMS +# Streamlit app GitHub repository name (to download artifact from). +ARG GITHUB_REPO=streamlit-template + + +# Step 1: set up a sane build system +USER root + +RUN apt-get -y update +# note: streamlit in docker needs libgtk2.0-dev (see https://yugdamor.medium.com/importerror-libgthread-2-0-so-0-cannot-open-shared-object-file-no-such-file-or-directory-895b94a7827b) +RUN apt-get install -y --no-install-recommends --no-install-suggests wget ca-certificates libgtk2.0-dev curl jq cron nginx +RUN update-ca-certificates + +# Install Github CLI +RUN (type -p wget >/dev/null || (apt-get update && apt-get install wget -y)) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt-get update \ + && apt-get install gh -y + +# Download and install miniforge. +ENV PATH="/root/miniforge3/bin:${PATH}" +RUN wget -q \ + https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh \ + && bash Miniforge3-Linux-x86_64.sh -b \ + && rm -f Miniforge3-Linux-x86_64.sh +RUN mamba --version + +# Make /root traversable so the entrypoint can `source +# /root/miniforge3/bin/activate ...` when the container runs as a non-root +# user (apptainer/singularity maps the host UID into the container; the +# default ubuntu /root is 0700 which would block path traversal). +x only, +# not +r, so the directory listing remains private. +RUN chmod o+x /root + +# Setup mamba environment. +RUN mamba create -n streamlit-env python=3.10 +RUN echo "mamba activate streamlit-env" >> ~/.bashrc +SHELL ["/bin/bash", "--rcfile", "~/.bashrc"] +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +#################################### install streamlit +# install packages +COPY requirements.txt requirements.txt +RUN mamba install pip +RUN python -m pip install --upgrade pip +RUN python -m pip install -r requirements.txt + +# Pre-create bind-mount targets so apptainer/singularity has a real attach +# point. Docker auto-creates missing `-v` targets, but singularity uses a +# read-only underlay and silently ignores `:rw` when the target isn't a +# real directory in the SIF β€” writes then fail with EROFS even though the +# host bind path is writable. +RUN mkdir -p /workspaces-streamlit-template /mounted-data + +# create workdir and copy over all streamlit related files/folders +WORKDIR /app +# note: specifying folder with slash as suffix and repeating the folder name seems important to preserve directory structure +WORKDIR /app +COPY assets/ /app/assets +COPY content/ /app/content +COPY docs/ /app/docs +COPY example-data/ /app/example-data +COPY gdpr_consent/ /app/gdpr_consent +COPY hooks/ /app/hooks +COPY src/ /app/src +COPY utils/ /app/utils +COPY app.py /app/app.py +COPY settings.json /app/settings.json +COPY default-parameters.json /app/default-parameters.json +COPY presets.json /app/presets.json + +# For streamlit configuration +COPY .streamlit/ /app/.streamlit/ + +COPY clean-up-workspaces.py /app/clean-up-workspaces.py + +# add cron job to the crontab +RUN echo "0 3 * * * /root/miniforge3/envs/streamlit-env/bin/python /app/clean-up-workspaces.py >> /app/clean-up-workspaces.log 2>&1" | crontab - + +# Number of Streamlit server instances for load balancing (default: 1 = no load balancer) +# Set to >1 to enable nginx load balancer with multiple Streamlit instances +ENV STREAMLIT_SERVER_COUNT=1 + +# Install the apptainer-compatible entrypoint (shared with the full image). +# The script auto-skips the Redis/RQ section when redis-server is not +# installed, so it works equally well in the simple variant. +COPY docker/entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh + +# Patch Analytics +RUN mamba run -n streamlit-env python hooks/hook-analytics.py + +# Set Online Deployment +RUN jq '.online_deployment = true' settings.json > tmp.json && mv tmp.json settings.json + +# Download latest OpenMS App executable as a ZIP file. +# ARG declared here (not at the top) β€” otherwise the per-run token busts the cache. +ARG GITHUB_TOKEN +RUN if [ -n "$GITHUB_TOKEN" ]; then \ + echo "GITHUB_TOKEN is set, proceeding to download the release asset..."; \ + gh release download -R ${GITHUB_USER}/${GITHUB_REPO} -p "OpenMS-App.zip" -D /app; \ + else \ + echo "GITHUB_TOKEN is not set, skipping the release asset download."; \ + fi + +# make sure that mamba environment is used +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +EXPOSE $PORT +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/Dockerfile_simple.arm b/Dockerfile_simple.arm new file mode 100644 index 0000000..be57317 --- /dev/null +++ b/Dockerfile_simple.arm @@ -0,0 +1,127 @@ +# This Dockerfile creates a container with pyOpenMS +# It also adds a basic streamlit server that serves a pyOpenMS-based app. +# hints: +# build image with: docker build -f Dockerfile_simple.arm --no-cache -t streamlitapp:latest-arm64 --build-arg GITHUB_TOKEN= . 2>&1 | tee build.log +# check if image was build: docker image ls +# run container: docker run -p 8501:8501 streamlitapp:latest +# debug container after build (comment out ENTRYPOINT) and run container with interactive /bin/bash shell +# prune unused images/etc. to free disc space (e.g. might be needed on gitpod). Use with care.: docker system prune --all --force + +FROM ubuntu:22.04 AS stage1 +ARG OPENMS_REPO=https://github.com/OpenMS/OpenMS.git +ARG OPENMS_BRANCH=develop +ARG PORT=8501 +# Streamlit app GitHub user name (to download artifact from). +ARG GITHUB_USER=OpenMS +# Streamlit app GitHub repository name (to download artifact from). +ARG GITHUB_REPO=streamlit-template + + +# Step 1: set up a sane build system +USER root + +RUN apt-get -y update +# note: streamlit in docker needs libgtk2.0-dev (see https://yugdamor.medium.com/importerror-libgthread-2-0-so-0-cannot-open-shared-object-file-no-such-file-or-directory-895b94a7827b) +RUN apt-get install -y --no-install-recommends --no-install-suggests wget ca-certificates libgtk2.0-dev curl jq cron nginx +RUN update-ca-certificates + +# Install Github CLI +RUN (type -p wget >/dev/null || (apt-get update && apt-get install wget -y)) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt-get update \ + && apt-get install gh -y + +# Download and install miniforge. +ENV PATH="/root/miniforge3/bin:${PATH}" +RUN wget -q \ + https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh \ + && bash Miniforge3-Linux-aarch64.sh -b \ + && rm -f Miniforge3-Linux-aarch64.sh +RUN mamba --version + +# Make /root traversable so the entrypoint can `source +# /root/miniforge3/bin/activate ...` when the container runs as a non-root +# user (apptainer/singularity maps the host UID into the container; the +# default ubuntu /root is 0700 which would block path traversal). +x only, +# not +r, so the directory listing remains private. +RUN chmod o+x /root + +# Setup mamba environment. +RUN mamba create -n streamlit-env python=3.10 +RUN echo "mamba activate streamlit-env" >> ~/.bashrc +SHELL ["/bin/bash", "--rcfile", "~/.bashrc"] +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +#################################### install streamlit +# install packages +COPY requirements.txt requirements.txt +RUN mamba install pip +RUN python -m pip install --upgrade pip +RUN python -m pip install -r requirements.txt + +# Pre-create bind-mount targets so apptainer/singularity has a real attach +# point. Docker auto-creates missing `-v` targets, but singularity uses a +# read-only underlay and silently ignores `:rw` when the target isn't a +# real directory in the SIF β€” writes then fail with EROFS even though the +# host bind path is writable. +RUN mkdir -p /workspaces-streamlit-template /mounted-data + +# create workdir and copy over all streamlit related files/folders +WORKDIR /app +# note: specifying folder with slash as suffix and repeating the folder name seems important to preserve directory structure +WORKDIR /app +COPY assets/ /app/assets +COPY content/ /app/content +COPY docs/ /app/docs +COPY example-data/ /app/example-data +COPY gdpr_consent/ /app/gdpr_consent +COPY hooks/ /app/hooks +COPY src/ /app/src +COPY utils/ /app/utils +COPY app.py /app/app.py +COPY settings.json /app/settings.json +COPY default-parameters.json /app/default-parameters.json +COPY presets.json /app/presets.json + +# For streamlit configuration +COPY .streamlit/ /app/.streamlit/ + +COPY clean-up-workspaces.py /app/clean-up-workspaces.py + +# add cron job to the crontab +RUN echo "0 3 * * * /root/miniforge3/envs/streamlit-env/bin/python /app/clean-up-workspaces.py >> /app/clean-up-workspaces.log 2>&1" | crontab - + +# Number of Streamlit server instances for load balancing (default: 1 = no load balancer) +# Set to >1 to enable nginx load balancer with multiple Streamlit instances +ENV STREAMLIT_SERVER_COUNT=1 + +# Install the apptainer-compatible entrypoint (shared with the full image). +# The script auto-skips the Redis/RQ section when redis-server is not +# installed, so it works equally well in the simple variant. +COPY docker/entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh + +# Patch Analytics +RUN mamba run -n streamlit-env python hooks/hook-analytics.py + +# Set Online Deployment +RUN jq '.online_deployment = true' settings.json > tmp.json && mv tmp.json settings.json + +# Download latest OpenMS App executable as a ZIP file. +# ARG declared here (not at the top) β€” otherwise the per-run token busts the cache. +ARG GITHUB_TOKEN +RUN if [ -n "$GITHUB_TOKEN" ]; then \ + echo "GITHUB_TOKEN is set, proceeding to download the release asset..."; \ + gh release download -R ${GITHUB_USER}/${GITHUB_REPO} -p "OpenMS-App.zip" -D /app; \ + else \ + echo "GITHUB_TOKEN is not set, skipping the release asset download."; \ + fi + +# make sure that mamba environment is used +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +EXPOSE $PORT +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/README.md b/README.md index 89f7d15..5a1db4b 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,203 @@ -# quantms-web β€” DDA Label-Free Quantification - -A browser-based **Data-Dependent Acquisition (DDA) Label-Free Quantification** workflow for proteomics. Upload mzML files and a protein FASTA; get identified and quantified proteins with volcano plots, PCA, and clustered heatmaps. No CLI, no Nextflow config. - -quantms-web mirrors the **dda-lfq branch of the [quantms Nextflow workflow](https://github.com/bigbio/quantms)** but runs as a [Streamlit](https://streamlit.io) app powered by OpenMS TOPP tools. - -## Pipeline - -| Stage | Tool | What it does | -|---|---|---| -| 1. Identification | Comet | Peptide-spectrum matching against a protein database | -| 2. Rescoring | Percolator | ML-based statistical validation of PSMs | -| 3. Filtering | IDFilter | FDR-controlled peptide identification filtering | -| 4. Quantification | ProteomicsLFQ | Label-free quantification across samples | -| 5. Analysis | Built-in | Volcano plots, PCA, heatmaps, spectral library export | - -## Run locally - -Install the Python dependencies and launch: +# OpenMS streamlit template + +[![Open Template!](https://static.streamlit.io/badges/streamlit_badge_black_white.svg)](https://abi-services.cs.uni-tuebingen.de/streamlit-template/) + +This repository contains a template app for OpenMS workflows in a web application using the **streamlit** framework. It serves as a foundation for apps ranging from simple workflows with **pyOpenMS** to complex workflows utilizing **OpenMS TOPP tools** with parallel execution. It includes solutions for handling user data and parameters in workspaces as well as deployment with docker-compose. + +## Features + +- Workspaces for user data with unique shareable IDs +- Persistent parameters and input files within a workspace +- local and online mode +- Captcha control +- Packaged executables for Windows +- framework for workflows with OpenMS TOPP tools +- Deployment [with docker-compose](https://github.com/OpenMS/streamlit-deployment) + +## πŸ”— Try the Online Demo + +Explore the hosted version here: πŸ‘‰ [Live App](https://abi-services.cs.uni-tuebingen.de/streamlit-template/) + +## πŸ’» Run Locally + +To run the app locally: + +1. **Clone the repository** + ```bash + git clone https://github.com/OpenMS/streamlit-template.git + cd streamlit-template + ``` + +2. **Install dependencies** + + Make sure you can run ```pip``` commands. + + Install all dependencies with: + ```bash + pip install -r requirements.txt + ``` + +4. **Launch the app** + ```bash + streamlit run app.py + ``` + +> ⚠️ Note: The local version offers limited functionality. Features that depend on OpenMS TOPP tools are only available out of the box in the Docker setup. For the local version [OpenMS Command Line Tools](https://openms.readthedocs.io/en/latest/about/installation.html) must be installed separately. + + +## 🐳 Build with Docker + +This repository contains two Dockerfiles. + +1. `Dockerfile`: This Dockerfile builds all dependencies for the app including Python packages and the OpenMS TOPP tools. Recommended for more complex workflows where you want to use the OpenMS TOPP tools for instance with the **TOPP Workflow Framework**. +2. `Dockerfile_simple`: This Dockerfile builds only the Python packages. Recommended for simple apps using pyOpenMS only. + +1. **Install Docker** + + Install Docker from the [official Docker installation guide](https://docs.docker.com/engine/install/) + +
+ Click to expand + + ```bash + # Remove older Docker versions (if any) + for pkg in docker.io docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do sudo apt-get remove -y $pkg; done + ``` + +
+ +2. **Test Docker** + + Verify that Docker is working. + ```bash + docker run hello-world + ``` + When running this command, you should see a hello world message from Docker. + +3. **Clone the repository** + ```bash + git clone https://github.com/OpenMS/streamlit-template.git + cd streamlit-template + ``` + +4. **Specify GitHub token (to download Windows executables).** + + Create a temporary `.env` file with your Github token. + + It should contain only one line: + `GITHUB_TOKEN=` + + ℹ️ **Note:** This step is not strictly required, but skipping it will remove the option to download executables from the WebApp. + +3. **Build & Launch the App** + + To build and start the containers. + From the project root directory: + + ```bash + docker-compose up -d --build + ``` + At the end, you should see this: + ``` + [+] Running 2/2 + βœ” openms-streamlit-template Built + βœ” Container openms-streamlit-template Started + ``` + + To make sure server started successfully, run `docker compose ps`. You should see `Up` status: + ``` + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 4abe0603e521 openms_streamlit_template "/app/entrypoint.sh …" 7 minutes ago Up 7 minutes 0.0.0.0:8501->8501/tcp, :::8501->8501/tcp openms-streamlit-template + ``` + + To map the port to default streamlit port `8501` and launch. + + ``` + docker run -p 8505:8501 openms_streamlit_template + ``` + + ### Mount a local data directory + + To make a directory of MS files on the host available to the running app + without uploading or copying them, bind-mount it into the container at + the path configured by `local_data_dir` in `settings.json` (the Docker + image defaults this to `/mounted-data`): + + ``` + docker run -p 8501:8501 \ + -v /path/on/host:/mounted-data:ro \ + openms_streamlit_template + ``` + + The upload widget auto-detects the mount: when the directory exists at + runtime it shows an in-app tree browser; selected files are referenced + in place via `external_files.txt` (no copy into the workspace volume), + so the mount can safely be read-only. Omitting `-v` hides the browser + and falls back to the standard upload UI. To use a different container + path, change `local_data_dir` in `settings.json` before building. + +## πŸ›°οΈ Run with Apptainer / Singularity (HPC) + +Apptainer (formerly Singularity) is the dominant container runtime on HPC +clusters. CI publishes prebuilt SIFs to GHCR via ORAS, so you can pull a +ready-to-run image with no on-the-fly OCIβ†’SIF conversion and run it as your +user β€” no root, no `--writable-tmpfs` required: ```bash -git clone https://github.com/OpenMS/quantms-web.git -cd quantms-web -pip install -r requirements.txt -streamlit run app.py +apptainer pull --name openms-streamlit-template.sif \ + oras://ghcr.io/openms/streamlit-template/sif:latest +apptainer run \ + --bind /path/to/data:/mounted-data:ro \ + --bind /path/to/workspaces:/workspaces-streamlit-template \ + openms-streamlit-template.sif ``` -The full pipeline runs locally once the [OpenMS Command Line Tools](https://openms.readthedocs.io/en/latest/about/installation.html) are on your `PATH` β€” they provide Comet, Percolator, ProteomicsLFQ, and the rest of the TOPP suite. With Python alone, the pyOpenMS-backed parts of the UI still work. - -## Run with Docker - -Ships OpenMS and the search engines together so the full pipeline works out of the box: - -```bash -docker-compose up -d --build +Available tags follow the same scheme as the Docker images: `latest`, +`main-full`, `main-simple`, `v*-full`, `v*-simple`, and per-commit SHAs. +If a tag hasn't been prebuilt yet (e.g. a PR branch), fall back to on-the-fly +conversion: `apptainer pull docker://ghcr.io/openms/streamlit-template:`. +Requires apptainer 1.1+ or singularity-ce 3.10+ for the `oras://` transport. + +The entrypoint auto-detects the read-only root filesystem (set by apptainer's +default isolation) and switches its runtime state β€” Redis data directory, +nginx config, PID files β€” to `/tmp/openms-runtime-$$`, which is always +writable inside an apptainer container. The workspace cleanup cron job is +skipped in this mode; rerun `clean-up-workspaces.py` manually if needed. + +## βš–οΈ Legal pages (Impressum, Privacy Policy, Terms of Use) + +Every page shows **Impressum**, **Privacy Policy** and **Terms of Use** links at +the bottom of the sidebar, and the GDPR consent banner links to the privacy +policy. By default these point to the centrally maintained official OpenMS pages +(`https://openms.de/impressum`, `/privacy`, `/terms`). + +If you self-host a fork, override them in `settings.json` β€” an Impressum must +name the **actual operator**, not OpenMS: + +```json +"legal_links": { + "impressum": "https://your-domain.example/impressum", + "privacy": "https://your-domain.example/privacy", + "terms": "https://your-domain.example/terms" +} ``` -Open http://localhost:8501. - -## Windows installer - -Download the latest `.msi` from [Releases](https://github.com/OpenMS/quantms-web/releases) and double-click to install. Standalone β€” no Python or Docker required. +Any link you omit falls back to its OpenMS default. The `privacy` URL is reused +for the consent banner's privacy-policy link, so consent and policy stay in sync. -## Workspaces +## Documentation -Every analysis session runs in an isolated **workspace** that persists inputs, parameters, and results. In online deployments the workspace ID is part of the URL, so runs are resumable and shareable. +Documentation for **users** and **developers** is included as pages in [this template app](https://abi-services.cs.uni-tuebingen.de/streamlit-template/), indicated by the πŸ“– icon. ## Citation -MΓΌller, T. D., Siraj, A., et al. *OpenMS WebApps: Building User-Friendly Solutions for MS Analysis.* Journal of Proteome Research (2025). [doi:10.1021/acs.jproteome.4c00872](https://doi.org/10.1021/acs.jproteome.4c00872) +Please cite: +MΓΌller, T. D., Siraj, A., et al. OpenMS WebApps: Building User-Friendly Solutions for MS Analysis. Journal of Proteome Research (2025). [https://doi.org/10.1021/acs.jproteome.4c00872](https://doi.org/10.1021/acs.jproteome.4c00872) ## References -- Pfeuffer, J., Bielow, C., Wein, S. et al. *OpenMS 3 enables reproducible analysis of large-scale mass spectrometry data.* Nat Methods 21, 365–367 (2024). [doi:10.1038/s41592-024-02197-7](https://doi.org/10.1038/s41592-024-02197-7) -- RΓΆst HL, Schmitt U, Aebersold R, MalmstrΓΆm L. *pyOpenMS: a Python-based interface to the OpenMS mass-spectrometry algorithm library.* Proteomics 14, 74–77 (2014). [doi:10.1002/pmic.201300246](https://doi.org/10.1002/pmic.201300246) +- Pfeuffer, J., Bielow, C., Wein, S. et al. OpenMS 3 enables reproducible analysis of large-scale mass spectrometry data. Nat Methods 21, 365–367 (2024). [https://doi.org/10.1038/s41592-024-02197-7](https://doi.org/10.1038/s41592-024-02197-7) + +- RΓΆst HL, Schmitt U, Aebersold R, MalmstrΓΆm L. pyOpenMS: a Python-based interface to the OpenMS mass-spectrometry algorithm library. Proteomics. 2014 Jan;14(1):74-7. [https://doi.org/10.1002/pmic.201300246](https://doi.org/10.1002/pmic.201300246). PMID: [24420968](https://pubmed.ncbi.nlm.nih.gov/24420968/). + + diff --git a/docker-compose.yml b/docker-compose.yml index e0a3e1c..889529e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,11 @@ services: - 8501:8501 volumes: - workspaces-streamlit-template:/workspaces-streamlit-template + # Optional: bind-mount a host directory of MS data files at the path + # that `local_data_dir` in settings.json points to (the Docker image + # defaults this to /mounted-data). When the directory exists at + # runtime, the upload page shows an in-app file browser for it. + # - /path/on/host:/mounted-data:ro environment: # Number of Streamlit server instances (default: 1 = no load balancer). # Set to >1 to enable nginx load balancing across multiple Streamlit instances. diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..65cbf8c --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,224 @@ +#!/bin/bash +# Container entrypoint for the OpenMS streamlit template. +# +# Works with both Docker (writable root FS, runs as root) and +# Apptainer/Singularity (read-only root FS, runs as the host user's UID). +# On HPC clusters apptainer is the dominant runtime; this script makes the +# image usable there without --writable-tmpfs. +set -e + +# Force the app directory regardless of how the container was invoked. +# `apptainer instance start` does not always honor the Docker WORKDIR, so +# `streamlit run app.py` would otherwise resolve against the host's CWD. +cd /app + +# Breadcrumbs β€” surfaced via apptainer instance .out/.err on failure, harmless +# in docker mode. Cheap to keep around for ongoing apptainer support. +echo "entrypoint: uid=$(id -u) gid=$(id -g) cwd=$(pwd) host=$(hostname) tty=$(tty 2>/dev/null || echo none)" +echo "entrypoint: APPTAINER_NAME=${APPTAINER_NAME:-unset} SINGULARITY_NAME=${SINGULARITY_NAME:-unset} APPTAINER_CONTAINER=${APPTAINER_CONTAINER:-unset}" + +source /root/miniforge3/bin/activate streamlit-env +echo "entrypoint: conda env activated, streamlit=$(command -v streamlit || echo NOT_FOUND)" + +# ----------------------------------------------------------------------------- +# Apptainer / read-only root filesystem detection +# ----------------------------------------------------------------------------- +# Apptainer sets APPTAINER_NAME (and SINGULARITY_NAME for backwards compat). +# As a fallback we probe /var/run for writability: docker = writable, apptainer +# default = read-only. Either signal flips us into "read-only mode". +if [ -n "${APPTAINER_NAME:-}" ] || [ -n "${SINGULARITY_NAME:-}" ] \ + || [ -n "${APPTAINER_CONTAINER:-}" ] || [ -n "${SINGULARITY_CONTAINER:-}" ] \ + || ! ( : > /var/run/.openms-rw-probe ) 2>/dev/null; then + READONLY_ROOT=1 + echo "Detected read-only root filesystem (apptainer/singularity mode)" +else + READONLY_ROOT=0 + rm -f /var/run/.openms-rw-probe 2>/dev/null || true +fi + +# Pick state paths. In read-only mode we must use /tmp (always tmpfs in +# apptainer); in docker mode we keep the conventional /var paths so existing +# docker-compose / k8s deployments are unaffected. +if [ "$READONLY_ROOT" -eq 1 ]; then + RUNTIME_DIR="${OPENMS_RUNTIME_DIR:-/tmp/openms-runtime-$$}" + mkdir -p "$RUNTIME_DIR" + REDIS_DATA_DIR="$RUNTIME_DIR/redis" + REDIS_PID_FILE="$RUNTIME_DIR/redis.pid" + # Apptainer/singularity share the host's network namespace by default. If + # the host has anything listening on 6379 (a system redis-server, a docker + # container, a previous singularity instance that didn't clean up), our + # `redis-server --daemonize` silently fails with EADDRINUSE and the local + # redis-cli ping happily connects to the host's redis instead β€” which + # leaves stale `worker-1` records lying around and ultimately runs the + # workflow's mkdir outside our mount namespace (no bind β†’ EROFS). A unix + # socket sidesteps the network stack entirely; the path is unambiguously + # ours. + REDIS_SOCKET="$RUNTIME_DIR/redis.sock" + REDIS_URL="unix://${REDIS_SOCKET}" + export REDIS_URL + NGINX_CONF_DIR="$RUNTIME_DIR/nginx" + NGINX_PID_FILE="$RUNTIME_DIR/nginx.pid" + mkdir -p "$REDIS_DATA_DIR" "$NGINX_CONF_DIR" + # Marker for out-of-band discovery (e.g. `apptainer exec ... redis-cli` + # from CI). The entrypoint's exported env doesn't propagate to fresh + # exec invocations, so write the resolved URL to a stable path. + echo "$REDIS_URL" > /tmp/openms-redis-url 2>/dev/null || true +else + RUNTIME_DIR="/var/run" + REDIS_DATA_DIR="/var/lib/redis" + REDIS_PID_FILE="/var/run/redis.pid" + REDIS_SOCKET="" + NGINX_CONF_DIR="/etc/nginx" + NGINX_PID_FILE="/run/nginx.pid" +fi + +# ----------------------------------------------------------------------------- +# Workspace cleanup cron (best-effort) +# ----------------------------------------------------------------------------- +# `service cron start` writes /var/run/crond.pid; it cannot work on a read-only +# root. The cleanup job is optional β€” workspaces just accumulate until the +# container is rebuilt, which is acceptable for HPC use cases where users +# manage their own workspace volumes. +if [ "$READONLY_ROOT" -eq 0 ]; then + service cron start || echo "WARN: cron failed to start; workspace cleanup disabled" +else + echo "Skipping cron (read-only root); run clean-up-workspaces.py manually if needed" +fi + +# ----------------------------------------------------------------------------- +# Redis + RQ workers (only present in the full image) +# ----------------------------------------------------------------------------- +# The simple image does not install redis-server. Skip the whole queue section +# when the binary is missing, so this entrypoint can be shared by both images. +if command -v redis-server >/dev/null 2>&1; then + if [ -n "$REDIS_SOCKET" ]; then + echo "Starting Redis server (data=$REDIS_DATA_DIR, socket=$REDIS_SOCKET)..." + # --port 0 disables the TCP listener entirely β€” we only accept the + # unix socket. This is the whole point of switching to a socket in + # apptainer mode: the host's network namespace (shared by default) + # cannot conflict with us, and there is no fall-through to a stray + # host redis-server. + redis-server --daemonize yes \ + --dir "$REDIS_DATA_DIR" \ + --pidfile "$REDIS_PID_FILE" \ + --unixsocket "$REDIS_SOCKET" \ + --unixsocketperm 700 \ + --port 0 \ + --appendonly no + REDIS_CLI_ARGS=(-s "$REDIS_SOCKET") + else + echo "Starting Redis server (data=$REDIS_DATA_DIR)..." + redis-server --daemonize yes \ + --dir "$REDIS_DATA_DIR" \ + --pidfile "$REDIS_PID_FILE" \ + --appendonly no + REDIS_CLI_ARGS=() + fi + + # Bounded wait so a broken redis-server (e.g. socket can't be created or + # an unexpected fork failure) fails the container fast instead of hanging + # forever and never serving /_stcore/health. + REDIS_STARTUP_RETRIES="${REDIS_STARTUP_RETRIES:-30}" + for i in $(seq 1 "$REDIS_STARTUP_RETRIES"); do + if redis-cli "${REDIS_CLI_ARGS[@]}" ping >/dev/null 2>&1; then + echo "Redis is ready" + break + fi + echo "Waiting for Redis... attempt $i/$REDIS_STARTUP_RETRIES" + sleep 1 + done + if ! redis-cli "${REDIS_CLI_ARGS[@]}" ping >/dev/null 2>&1; then + echo "ERROR: Redis failed to become ready within ${REDIS_STARTUP_RETRIES}s" >&2 + exit 1 + fi + + WORKER_COUNT="${RQ_WORKER_COUNT:-1}" + echo "Starting $WORKER_COUNT RQ worker(s)..." + for i in $(seq 1 "$WORKER_COUNT"); do + rq worker openms-workflows --url "$REDIS_URL" --name "worker-$i" & + done +fi + +# ----------------------------------------------------------------------------- +# Streamlit (single instance or behind nginx load balancer) +# ----------------------------------------------------------------------------- +SERVER_COUNT="${STREAMLIT_SERVER_COUNT:-1}" + +# Surface a misconfigured opt-in to load balancing β€” silently downgrading to a +# single instance has bitten users on the simple image variant where nginx +# isn't installed. +if [ "$SERVER_COUNT" -gt 1 ] && ! command -v nginx >/dev/null 2>&1; then + echo "WARN: STREAMLIT_SERVER_COUNT=$SERVER_COUNT requested but nginx is not installed (simple image?); falling back to a single instance" >&2 +fi + +if [ "$SERVER_COUNT" -gt 1 ] && command -v nginx >/dev/null 2>&1; then + echo "Starting $SERVER_COUNT Streamlit instances with nginx load balancer..." + + UPSTREAM_SERVERS="" + BASE_PORT=8510 + for i in $(seq 0 $((SERVER_COUNT - 1))); do + PORT=$((BASE_PORT + i)) + UPSTREAM_SERVERS="${UPSTREAM_SERVERS} server 127.0.0.1:${PORT}; +" + done + + NGINX_CONF_FILE="$NGINX_CONF_DIR/nginx.conf" + cat > "$NGINX_CONF_FILE" <` + ```python + workspaces_directory = Path("/workspaces-streamlit-template") + ``` +3. Update `README.md` accordingly + + +**Dockerfile-related** +1. Choose one of the Dockerfiles depending on your use case: + - `Dockerfile` builds OpenMS including TOPP tools + - `Dockerfile_simple` uses pyOpenMS only +2. Update the Dockerfile: + - with the `GITHUB_USER` owning the Streamlit app repository + - with the `GITHUB_REPO` name of the Streamlit app repository + - if your main page Python file is not called `app.py`, modify the following line + ```dockerfile + RUN echo "mamba run --no-capture-output -n streamlit-env streamlit run app.py" >> /app/entrypoint.sh + ``` +3. Update Python package dependency files: + - `requirements.txt` if using `Dockerfile_simple` + - `environment.yml` if using `Dockerfile` + +## How to build a workflow + +### Simple workflow using pyOpenMS + +Take a look at the example pages `Simple Workflow` or `Workflow with mzML files` for examples (on the *sidebar*). Put Streamlit logic inside the pages and call the functions with workflow logic from from the `src` directory (for our examples `src/simple_workflow.py` and `src/mzmlfileworkflow.py`). + +### Complex workflow using TOPP tools + +This template app features a module in `src/workflow` that allows for complex and long workflows to be built very efficiently. Check out the `TOPP Workflow Framework` page for more information (on the *sidebar*). + +For building **conditional parameter UI** (widgets that appear or disappear based on another parameter's value), see the *Reactive parameters* subsection of the `TOPP Workflow Framework` page's *Parameter Input* section. diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..86653d1 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,100 @@ +# OpenMS streamlit app deployment + +OpenMS streamlit apps can be deployed two ways: + +- **Kubernetes** β€” Kustomize manifests under `k8s/` with CI-built images pushed to GHCR. See the "Developers Guide: Kubernetes Deployment" page. +- **Docker Compose** β€” described below. Uses the external [OpenMS/streamlit-deployment](https://github.com/OpenMS/streamlit-deployment) repo to aggregate multiple apps as submodules. + +If you're using Claude Code, the `configure-app-settings` and `configure-docker-compose-deployment` skills automate the docker-compose path; `configure-app-settings` + `configure-k8s-deployment` automate the Kubernetes path. + +--- + +## Docker Compose + +Multiple streamlit apps based on the [OpenMS streamlit template](https://github.com/OpenMS/streamlit-template/) can be deployed together using docker compose. + +## Features + +- deploy all OpenMS apps at once +- user data (in workspaces) is stored in persistent docker volumes for each app + +## Requirements +- Docker Compose + +## Deployment (e.g., needed after one app changed) + +**1. Make sure submodules are up-to-data.** + +`git submodule init` + +`git submodule update` + +**2. Specify GitHub token (to download Windows executables).** + +> This is **important**! Omitting this step while result in all apps not having the option to download executables any more. + +Create a temporary `.env` file with your Github token. It should contain only one line: + +`GITHUB_TOKEN=` + +**3. Run docker-compose.** + +`docker-compose up --build -d` + +> Make sure to remove the `.env` file with your Github token after successful build + +## Add new app + +This will add your app as a submodule to the streamlit deployment repository. + +**1. Enable online mode in the apps settings.json.** + +**2. Fork and clone the [OpenMS streamlit deployment](https://github.com/OpenMS/streamlit-deployment) repository locally.** + +**3. Add your app as submodule. Make sure the app name is not used already.** + +`git submodule add ` + +**4. Initialize and update submodules.** + +`git submodule init` + +`git submodule update` + +**5. Add your app to `docker-compose.yml` file as a new service.** + +Copy the last service as a template. + +Check and update the following entries: + +- name of the service + - the name of the submodule +- build context + - the relative path to the submodule +- build dockerfile + - the correct Dockerfile +- image + - name of the docker image (typically the service name with underscores) +- ports + - chose an incremental host port number from the last service pointing to the streamlit port in docker container (8501) +- volumes + - update the names of the workspace directories, user data is stored outside of the docker container in a docker volume +- command + - update command with your main streamlit file + +**6. Test everything works locally.** + +Run docker-compose to launch all services. + +`docker-compose up --build -d` + +- there should be no errors building all services +- make sure all apps are accessible via their port from localhost +- test functionality of your app + +**7. Make a pull request with your changes to OpenMS/streamlit-deployment main branch.** + + + +# Other Architectures +In principle OpenMS runs on most processor architectures. The images are provided and tested for x86 but OpenMS can also be compiled on architectures like arm64. Please note that you might have to adjust the miniforge version according to the processor architecture. diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..06e0465 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,109 @@ +# Installation + +## Windows + +The app is available as pre-packaged Windows executable, including all dependencies. + +The windows executable is built by a GitHub action and can be downloaded [here](https://github.com/OpenMS/streamlit-template/actions/workflows/build-windows-executable-app.yaml). +Select the latest successfull run and download the zip file from the artifacts section, while signed in to GitHub. + +## Python + +Clone the [streamlit-template repository](https://github.com/OpenMS/streamlit-template). It includes files to install dependencies via pip or conda. + +## πŸ’» Run Locally + +To run the app locally: + +1. **Clone the repository** + ```bash + git clone https://github.com/OpenMS/streamlit-template.git + cd streamlit-template + ``` + +2. **Install dependencies** + + Make sure you can run ```pip``` commands. + + Install all dependencies with: + ```bash + pip install -r requirements.txt + ``` + +4. **Launch the app** + ```bash + streamlit run app.py + ``` + +> ⚠️ Note: The local version offers limited functionality. Features that depend on OpenMS are only available in the Docker setup. + + +## 🐳 Build with Docker + +This repository contains two Dockerfiles. + +1. `Dockerfile`: This Dockerfile builds all dependencies for the app including Python packages and the OpenMS TOPP tools. Recommended for more complex workflows where you want to use the OpenMS TOPP tools for instance with the **TOPP Workflow Framework**. +2. `Dockerfile_simple`: This Dockerfile builds only the Python packages. Recommended for simple apps using pyOpenMS only. + +1. **Install Docker** + + Install Docker from the [official Docker installation guide](https://docs.docker.com/engine/install/) + +
+ Click to expand + + ```bash + # Remove older Docker versions (if any) + for pkg in docker.io docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do sudo apt-get remove -y $pkg; done + ``` + +
+ +2. **Test Docker** + + Verify that Docker is working. + ```bash + docker run hello-world + ``` + When running this command, you should see a hello world message from Docker. + +3. **Clone the repository** + ```bash + git clone https://github.com/OpenMS/streamlit-template.git + cd streamlit-template + ``` + +4. **Specify GitHub token (to download Windows executables).** + + Create a temporary `.env` file with your Github token. + + It should contain only one line: + `GITHUB_TOKEN=` + + ℹ️ **Note:** This step is not strictly required, but skipping it will remove the option to download executables from the WebApp. + +3. **Build & Launch the App** + + To build and start the containers. + From the project root directory: + + ```bash + docker-compose up -d --build + ``` + At the end, you should see this: + ``` + [+] Running 2/2 + βœ” openms-streamlit-template Built + βœ” Container openms-streamlit-template Started + ``` + + To make sure server started successfully, run `docker compose ps`. You should see `Up` status: + ``` + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 4abe0603e521 openms_streamlit_template "/app/entrypoint.sh …" 7 minutes ago Up 7 minutes 0.0.0.0:8501->8501/tcp, :::8501->8501/tcp openms-streamlit-template + ``` + + To map the port to default streamlit port `8501` and launch. + + ``` + docker run -p 8505:8501 openms_streamlit_template diff --git a/docs/toppframework.py b/docs/toppframework.py new file mode 100644 index 0000000..a91b2be --- /dev/null +++ b/docs/toppframework.py @@ -0,0 +1,348 @@ +import streamlit as st +from src.Workflow import Workflow +from src.workflow.StreamlitUI import StreamlitUI +from src.workflow.FileManager import FileManager +from src.workflow.CommandExecutor import CommandExecutor +from src.workflow.ParameterManager import ParameterManager +from inspect import getsource + +def content(): + st.title("TOPP Workflow Framework Documentation") + + st.markdown( + """ +## Features + +- streamlined methods for uploading files, setting parameters, and executing workflows +- automatic parameter handling +- quickly build parameter interface for TOPP tools with all parameters from *ini* files +- automatically create a log file for each workflow run with stdout and stderr +- workflow output updates automatically in short intervalls +- user can leave the app and return to the running workflow at any time +- quickly build a workflow with multiple steps channelling files between steps +""" + ) + + st.markdown( + """ +## Quickstart + +This repository contains a module in `src/workflow` that provides a framework for building and running analysis workflows. + +The `WorkflowManager` class provides the core workflow logic. It uses the `Logger`, `FileManager`, `ParameterManager`, and `CommandExecutor` classes to setup a complete workflow logic. + +To build your own workflow edit the file `src/TOPPWorkflow.py`. Use any streamlit components such as tabs (as shown in example), columns, or even expanders to organize the helper functions for displaying file upload and parameter widgets. + +> πŸ’‘ Simply set a name for the workflow and overwrite the **`upload`**, **`configure`**, **`execution`** and **`results`** methods in your **`Workflow`** class. + +The file `content/6_TOPP-Workflow.py` displays the workflow content and can, but does not have to be modified. + +The `Workflow` class contains four important members, which you can use to build your own workflow: + +> **`self.params`:** dictionary of parameters stored in a JSON file in the workflow directory. Parameter handling is done automatically. Default values are defined in input widgets and non-default values are stored in the JSON file. + +> **`self.ui`:** object of type `StreamlitUI` contains helper functions for building the parameter and file upload widgets. + +> **`self.executor`:** object of type `CommandExecutor` can be used to run any command line tool alone or in parallel and includes a convenient method for running TOPP tools. + +> **`self.logger`:** object of type `Logger` to write any output to a log file during workflow execution. + +> **`self.file_manager`:** object of type `FileManager` to handle file types and creation of output directories. +""" + ) + + with st.expander("**Complete example for custom Workflow class**", expanded=False): + st.code(getsource(Workflow)) + + st.markdown( + """ +## File Upload + +All input files for the workflow will be stored within the workflow directory in the subdirectory `input-files` within it's own subdirectory for the file type. + +The subdirectory name will be determined by a **key** that is defined in the `self.ui.upload_widget` method. The uploaded files are available by the specific key for parameter input widgets and accessible while building the workflow. + +Calling this method will create a complete file upload page with the following components: + +- file uploader +- list of currently uploaded files with this key (or a warning if there are none) +- button to delete all files + +Fallback files(s) can be specified, which will be used if the user doesn't upload any files. This can be useful for example for database files where a default is provided. +""" + ) + + st.code(getsource(Workflow.upload)) + + st.info( + "πŸ’‘ Use the same **key** for parameter widgets, to select which of the uploaded files to use for analysis." + ) + + with st.expander("**Code documentation:**", expanded=True): + st.help(StreamlitUI.upload_widget) + + st.markdown( + """ +## Parameter Input + +The parameter page is already pre-defined as a form with buttons to **save parameters** and **load defaults** and a toggle to show TOPP tool parameters marked as advanced. + +Generating parameter input widgets is done with the `self.ui.input` method for any parameter and the `self.ui.input_TOPP` method for TOPP tools. + +**1. Choose `self.ui.input_widget` for any parameter not-related to a TOPP tool or `self.ui.select_input_file` for any input file:** + +It takes the obligatory **key** parameter. The key is used to access the parameter value in the workflow parameters dictionary `self.params`. Default values do not need to be specified in a separate file. Instead they are determined from the widgets default value automatically. Widget types can be specified or automatically determined from **default** and **options** parameters. It's suggested to add a **help** text and other parameters for numerical input. + +Make sure to match the **key** of the upload widget when calling `self.ui.input_TOPP`. + +**2. Choose `self.ui.input_TOPP` to automatically generate complete input sections for a TOPP tool:** + +It takes the obligatory **topp_tool_name** parameter and generates input widgets for each parameter present in the **ini** file (automatically created) except for input and output file parameters. For all input file parameters a widget needs to be created with `self.ui.select_input_file` with an appropriate **key**. For TOPP tool parameters only non-default values are stored. + +**3. Choose `self.ui.input_python` to automatically generate complete input sections for a custom Python tool:** + +Takes the obligatory **script_file** argument. The default location for the Python script files is in `src/python-tools` (in this case the `.py` file extension is optional in the **script_file** argument), however, any other path can be specified as well. Parameters need to be specified in the Python script in the **DEFAULTS** variable with the mandatory **key** and **value** parameters. +""" + ) + + with st.expander( + "Options to use as dictionary keys for parameter definitions (see `src/python-tools/example.py` for an example)" + ): + st.markdown( + """ +**Mandatory** keys for each parameter +- *key:* a unique identifier +- *value:* the default value + +**Optional** keys for each parameter +- *name:* the name of the parameter +- *hide:* don't show the parameter in the parameter section (e.g. for **input/output files**) +- *options:* a list of valid options for the parameter +- *min:* the minimum value for the parameter (int and float) +- *max:* the maximum value for the parameter (int and float) +- *step_size:* the step size for the parameter (int and float) +- *help:* a description of the parameter +- *widget_type:* the type of widget to use for the parameter (default: auto) +- *advanced:* whether or not the parameter is advanced (default: False) +""" + ) + + st.code(getsource(Workflow.configure)) + st.info( + "πŸ’‘ Access parameter widget values by their **key** in the `self.params` object, e.g. `self.params['mzML-files']` will give all selected mzML files." + ) + + st.markdown( + """ +### Reactive parameters (conditional UI) + +By default every parameter widget is rendered inside an isolated `st.fragment`: changing it reruns only that widget, which keeps large parameter sections fast but also means a widget **cannot** show or hide other widgets. Pass **`reactive=True`** to `self.ui.input_widget`, `self.ui.select_input_file` or `self.ui.input_TOPP` to render the widget directly in the parent `configure()` scope instead β€” a change then reruns `configure()`, so you can conditionally render downstream widgets based on the current value. + +Read the current value from `st.session_state` (**not** `self.params`, which is only loaded once per run and is stale within the same rerun) using the parameter-manager prefixes: `param_prefix` for `input_widget` / `select_input_file` keys, and `topp_param_prefix` for TOPP keys of the form `":1:"` (the `` matches the tool's ini, as shown for each widget in the panel). +""" + ) + st.code( + '''@st.fragment +def configure(self) -> None: + pm = self.parameter_manager + + # A custom widget that reveals more widgets when checked + self.ui.input_widget( + "advanced-mode", False, "Advanced mode", + widget_type="checkbox", reactive=True, + ) + if st.session_state.get(f"{pm.param_prefix}advanced-mode", False): + self.ui.input_widget("threads", 4, "Threads", widget_type="number") + + # A TOPP parameter driving conditional UI (e.g. TMT type -> channel count) + self.ui.input_TOPP("IsobaricAnalyzer", reactive=True) + iso_type = st.session_state.get(f"{pm.topp_param_prefix}IsobaricAnalyzer:1:type", "") + if iso_type.startswith("tmt"): + self.ui.input_widget("tmt-channels", 10, "TMT channels", widget_type="number")''' + ) + st.info( + "πŸ’‘ `reactive=True` is opt-in per widget and defaults to `False`. On `input_TOPP` it un-isolates the whole tool panel, so **any** parameter change reruns `configure()` β€” leave it off unless another widget's visibility depends on a value in that panel." + ) + + with st.expander("**Code documentation**", expanded=True): + st.help(StreamlitUI.input_widget) + st.help(StreamlitUI.select_input_file) + st.help(StreamlitUI.input_TOPP) + st.help(StreamlitUI.input_python) + + st.markdown( + """ +## Parameter Presets + +Presets provide a way to offer users quick parameter configuration options for common analysis scenarios. + +### Enabling Presets + +1. Create a `presets.json` file at the repository root +2. Add preset definitions for your workflow (workflow name must be normalized: lowercase with hyphens) +3. Presets automatically appear in the parameter section via `self.ui.preset_buttons()` + +### presets.json Structure + +```json +{ + "your-workflow-name": { + "Preset Display Name": { + "_description": "Tooltip text for the button", + "TOPPToolName": { + "algorithm:param:path": value + }, + "_general": { + "custom_widget_key": value + } + } + } +} +``` + +### Key Points + +- **Workflow matching**: Workflow name is normalized (lowercase, hyphens for spaces). "TOPP Workflow" β†’ "topp-workflow" +- **TOPP parameters**: Use the full parameter path (e.g., `algorithm:common:noise_threshold_int`) +- **General parameters**: Use `_general` for custom `input_widget` parameters +- **Descriptions**: Keys prefixed with `_` are metadata (not applied as parameters) +- **Opt-in feature**: No `presets.json` = no buttons shown (backward compatible) +""" + ) + + with st.expander("**Code documentation**", expanded=True): + st.help(StreamlitUI.preset_buttons) + st.help(ParameterManager.load_presets) + st.help(ParameterManager.apply_preset) + + st.markdown( + """ +## Building the Workflow + +Building the workflow involves **calling all (TOPP) tools** using **`self.executor`** with **input and output files** based on the **`FileManager`** class. For TOPP tools non-input-output parameters are handled automatically. Parameters for other processes and workflow logic can be accessed via widget keys (set on the parameter page) in the **`self.params`** dictionary. + +### FileManager + +The `FileManager` class serves as an interface for unified input and output files with useful functionality specific to building workflows, such as **setting a (new) file type** and **subdirectory in the workflows result directory**. + +Use the **`get_files`** method to get a list of all file paths as strings. + +Optionally set the following parameters modify the files: + +- **set_file_type** (str): set new file types and result subdirectory. +- **set_results_dir** (str): set a new subdirectory in the workflows result directory. +- **collect** (bool): collect all files into a single list. Will return a list with a single entry, which is a list of all files. Useful to pass to tools which can handle multiple input files at once. +""" + ) + + st.code( + """ +# Get all file paths as strings from self.param entry. +mzML_files = self.file_manager.get_files(self.params["mzML-files]) +# mzML_files = ['../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Control.mzML', '../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Treatment.mzML'] + +# Creating output files for a TOPP tool, setting a new file type and result subdirectory name. +feature_detection_out = self.file_manager.get_files(mzML_files, set_file_type="featureXML", set_results_dir="feature-detection") +# feature_detection_out = ['../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Control.featureXML', '../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Treatment.featureXML'] + +# Setting a name for the output directory automatically (useful if you never plan to access these files within the results page). +feature_detection_out = self.file_manager.get_files(mzML_files, set_file_type="featureXML", set_results_dir="auto") +# feature_detection_out = ['../workspaces-streamlit-template/default/topp-workflow/results/6DUd/Control.featureXML', '../workspaces-streamlit-template/default/topp-workflow/results/6DUd/Treatment.featureXML'] + +# Combining all mzML files to be passed to a TOPP tool in a single run. Using "collected" files as argument for self.file_manager.get_files will "un-collect" them. +mzML_files = self.file_manager.get_files(mzML_files, collect=True) +# mzML_files = [['../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Control.mzML', '../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Treatment.mzML']] + """ + ) + + with st.expander("**Code documentation**", expanded=True): + st.help(FileManager.get_files) + + st.markdown( + """ +### Running commands + +It is possible to execute any command line command using the **`self.executor`** object, either a single command or a list of commands in parallel. Furthermore a method to run TOPP tools is included. + +**1. Single command** + +The `self.executor.run_command` method takes a single command as input and optionally logs stdout and stderr to the workflow log (default True). +""" + ) + + st.code( + """ +self.executor.run_command(["command", "arg1", "arg2", ...]) +""" + ) + + st.markdown( + """ +**2. Run multiple commands in parallel** + +The `self.executor.run_multiple_commands` method takes a list of commands as inputs. + +**3. Run TOPP tools** + +The `self.executor.run_topp` method takes a TOPP tool name as input and a dictionary of input and output files as input. The **keys** need to match the actual input and output parameter names of the TOPP tool. The **values** should be of type `FileManager`. All other **non-default parameters (from input widgets)** will be passed to the TOPP tool automatically. + +Depending on the number of input files, the TOPP tool will be run either in parallel or in a single run (using **`FileManager.collect`**). +""" + ) + + st.info( + """πŸ’‘ **Input and output file order** + +In many tools, a single input file is processed to produce a single output file. +When dealing with lists of input or output files, the convention is that +files are paired based on their order. For instance, the n-th input file is +assumed to correspond to the n-th output file, maintaining a structured +relationship between input and output data. +""" + ) + st.code( + """ +# e.g. FeatureFinderMetabo takes single input files +in_files = self.file_manager.get_files(["sample1.mzML", "sample2.mzML"]) +out_files = self.file_manager.get_files(in_files, set_file_type="featureXML", set_results_dir="feature-detection") + +# Run FeatureFinderMetabo tool with input and output files in parallel for each pair of input/output files. +self.executor.run_topp("FeatureFinderMetabo", input_output={"in": in_files, "out": out_files}) +# FeaturFinderMetabo -in sample1.mzML -out workspace-dir/results/feature-detection/sample1.featureXML +# FeaturFinderMetabo -in sample2.mzML -out workspace-dir/results/feature-detection/sample2.featureXML + +# Run SiriusExport tool with mutliple input and output files. +out = self.file_manager.get_files("sirius.ms", set_results_dir="sirius-export") +self.executor.run_topp("SiriusExport", {"in": self.file_manager.get_files(in_files, collect=True), + "in_featureinfo": self.file_manager.get_files(out_files, collect=True), + "out": out_se}) +# SiriusExport -in sample1.mzML sample2.mzML -in_featureinfo sample1.featureXML sample2.featureXML -out sirius.ms + """ + ) + + st.markdown( + """ +**4. Run custom Python scripts** + +Sometimes it is useful to run custom Python scripts, for example for extra functionality which is not included in a TOPP tool. + +`self.executor.run_python` works similar to `self.executor.run_topp`, but takes a single Python script as input instead of a TOPP tool name. The default location for the Python script files is in `src/python-tools` (in this case the `.py` file extension is optional in the **script_file** argument), however, any other path can be specified as well. Input and output file parameters need to be specified in the **input_output** dictionary. +""" + ) + + st.code( + """ +# e.g. example Python tool which modifies mzML files in place based on experimental design +self.ui.input_python(script_file="example", input_output={"in": in_mzML, "in_experimantal_design": FileManager(["path/to/experimantal-design.tsv"])}) + """ + ) + + st.markdown("**Example for a complete workflow:**") + + st.code(getsource(Workflow.execution)) + + with st.expander("**Code documentation**", expanded=True): + st.help(CommandExecutor.run_command) + st.help(CommandExecutor.run_multiple_commands) + st.help(CommandExecutor.run_topp) + st.help(CommandExecutor.run_python) \ No newline at end of file diff --git a/docs/user_guide.md b/docs/user_guide.md new file mode 100644 index 0000000..63bf521 --- /dev/null +++ b/docs/user_guide.md @@ -0,0 +1,77 @@ +# User Guide + +Welcome to the OpenMS Streamlit Web Application! This guide will help you understand how to use our tools effectively. + +## Advantages of OpenMS Web Apps + +OpenMS web applications provide a user-friendly interface for accessing the powerful features of OpenMS. Here are a few advantages: +- **Accessibility**: Access powerful OpenMS algorithms and TOPP tools from any device with a web browser. +- **Ease of Use**: Simplified user interface makes it easy for both beginners and experts to perform complex analyses. +- **No Installation Required**: Use the tools without the need to install OpenMS locally, saving time and system resources. + +## Workspaces + +In the OpenMS web application, workspaces are designed to keep your analysis organized: +- **Workspace Specific Parameters and Files**: Each workspace stores parameters and files (uploaded input files and results from workflows). +- **Persistence**: Your workspaces and parameters are saved, so you can return to your analysis anytime and pick up where you left off. Simply bookmark the page! + + +### File Uploads +- **Online Mode**: You can upload only one file at a time. This helps manage server load and optimizes performance. + +- **Local Mode**: Multiple file uploads are supported, giving you flexibility when working with large datasets. Additionally, the file size upload limit can be adjusted in the following ways: + 1. **Using `.streamlit/config.toml`**: + - You can modify the `.streamlit/config.toml` file and set the `maxUploadSize` parameter to your desired value. By default, this is set to 200MB. + - Example: + ```toml + [server] + maxUploadSize = 500 # Set the upload limit to 500MB + ``` + 2. **Using CLI Command**: + - You can customize the file size upload limit directly when running the application using the `--server.maxUploadSize` argument. + - Example: + ```bash + python run_app.py --server.maxUploadSize 500 + ``` + - This sets the upload limit to 500MB for the current session. + +- **Workspace Access**: + - In online mode, workspaces are stored temporarily and will be cleared after seven days of inactivity. + - In local mode, workspaces are saved on your local machine, allowing for persistent storage. Workspace directory can be specified in the `settings.json`. Defaults to `..` (parent directory). + +## Downloading Results + +You can download the results of your analyses, including data, figures and tables, directly from the application: +- **Figures**: Click the camera icon button, appearing while hovering on the top right corner of the figure. Set the desired image format in the settings panel in the side bar. +- **Tables**: Use the download button to save tables in *csv* format, appearing while hovering on the top right corner of the table. +- **Data**: Use the download section in the sidebar to download the raw results of your analysis. + +## Getting Started + +To get started: +1. Select or create a new workspace. +2. Upload your data file. +3. Set the necessary parameters for your analysis. +4. Run the analysis. +5. View and download your results. + +For more detailed information on each step, refer to the specific sections of this guide. + +## Parameter Presets + +Parameter presets allow you to quickly apply optimized parameter configurations for common analysis scenarios. When available, preset buttons appear on the parameter configuration page. + +### Using Presets + +1. Navigate to the parameter configuration page +2. Look for the **Parameter Presets** section below the "Show advanced parameters" toggle +3. Hover over a preset button to see its description +4. Click a preset to apply its optimized parameters +5. A confirmation message will appear when the preset is applied + +### What Presets Do + +- Presets override specific tool parameters with pre-configured values +- Only the parameters defined in the preset are changed; other parameters remain at their current values +- You can still modify individual parameters after applying a preset +- Use **Load default parameters** to reset all parameters to their original defaults \ No newline at end of file diff --git a/docs/win_exe_with_embed_py.md b/docs/win_exe_with_embed_py.md new file mode 100644 index 0000000..cb3d054 --- /dev/null +++ b/docs/win_exe_with_embed_py.md @@ -0,0 +1,278 @@ +## πŸ’» Create a window executable of a Streamlit App with embeddable Python + +To create an executable for Streamlit app on Windows, we'll use an embeddable version of Python.
+Here's a step-by-step guide: + +### Prerequisites + +You need a **system Python installation** (the regular Python installer from python.org) of the same version as the embeddable Python you'll download. This is required because the embeddable Python lacks development headers (`Python.h`) needed to compile native extensions. + +Install Python 3.11.9 from https://www.python.org/downloads/ if you don't have it already. + +### Download and Extract Python Embeddable Version + +1. Download a suitable Python embeddable version. For example, let's download Python 3.11.9: + + ```bash + # use curl command or manually download + curl -O https://www.python.org/ftp/python/3.11.9/python-3.11.9-embed-amd64.zip + ``` + +2. Extract the downloaded zip file: + + ```bash + mkdir python-3.11.9 + + unzip python-3.11.9-embed-amd64.zip -d python-3.11.9 + + rm python-3.11.9-embed-amd64.zip + ``` + +### Configure Python Environment + +1. Uncomment 'import site' in the `._pth` file: + + ```bash + # Uncomment to run site.main() automatically + # Remove hash from python-3.11.9/python311._pth file + import site + + # Or use command + sed -i '/^\s*#\s*import\s\+site/s/^#//' python-3.11.9/python311._pth + ``` + +### Install Required Packages + +Install all required packages from `requirements.txt` using the **system Python** with `--target` to install into the embeddable Python's site-packages: + +```bash +# Use system Python (which has development headers) to compile packages, +# installing into the embeddable Python's site-packages directory. +# The embeddable Python lacks Python.h headers needed for native extensions. +python -m pip install -r requirements.txt --target python-3.11.9/Lib/site-packages --upgrade --no-warn-script-location +``` + +> **Important**: Do NOT use `./python-3.11.9/python -m pip install ...` directly. The embeddable Python lacks the development headers required to compile native extensions (e.g., `Python.h`), which will cause builds to fail with errors like: +> ``` +> fatal error C1083: Cannot open include file: 'Python.h': No such file or directory +> ``` + +### Test and create `run_app.bat` file + +1. Test by running app + + ```batch + .\python-3.11.9\python -m streamlit run app.py + ``` + +2. Create a Clickable Shortcut + + Create a `run_app.bat` file to make running the app easier: + + ```batch + echo @echo off > run_app.bat + echo .\\python-3.11.9\\python -m streamlit run app.py >> run_app.bat + ``` + +### Create one executable folder + +1. Create a folder for your Streamlit app: + + ```bash + mkdir ../streamlit_exe + ``` + +2. Copy environment and app files: + + ```bash + # move Python environment folder + mv python-3.11.9 ../streamlit_exe + + # move run_app.bat file + mv run_app.bat ../streamlit_exe + + # copy streamlit app files + cp -r src pages .streamlit assets example-data ../streamlit_exe + cp app.py ../streamlit_exe + ``` + +3. Remove the server address from the bundled config to use `localhost` (default) instead of `0.0.0.0`, which doesn't work as a connect address on Windows: + + ```powershell + (Get-Content streamlit_exe/.streamlit/config.toml) -notmatch '^address' | Set-Content streamlit_exe/.streamlit/config.toml + ``` + +#### πŸš€ After successfully completing all these steps, the Streamlit app will be available by running the run_app.bat file. + +:pencil: You can still change the configuration of Streamlit app with .streamlit/config.toml file, e.g., provide a different port, change upload size, etc. + +## Build executable in github action automatically + +Automate the process of building executables for your project with the GitHub action example [Test streamlit executable for Windows with embeddable python](https://github.com/OpenMS/streamlit-template/blob/main/.github/workflows/test-win-exe-w-embed-py.yaml) +
+ +## Create MSI Installer using WiX Toolset + +After creating your executable folder, you can package it into an MSI installer using WiX Toolset. Here's how: + +### 1. Set Environment Variables + +Set these variables for consistent naming throughout the process: + +```batch +APP_NAME=OpenMS-StreamlitTemplateApp +APP_UpgradeCode= generate-new +``` + +To generate a new GUID for your application's UpgradeCode, you can use: + +- PowerShell: `[guid]::NewGuid().ToString()` +- Online GUID generator: https://www.guidgen.com/ +- Windows Command Prompt: `powershell -Command "[guid]::NewGuid().ToString()"` + +### 2. Install WiX Toolset + +1. Download WiX Toolset binaries: + ```batch + curl -LO https://github.com/wixtoolset/wix3/releases/download/wix3111rtm/wix311-binaries.zip + unzip wix311-binaries.zip -d wix + ``` + +### 3. Prepare Installation Files + +1. Create a SourceDir structure: + + ```batch + mkdir SourceDir + move streamlit_exe\* SourceDir + ``` + +2. Create Readme.txt: + + ```batch + # Create a Readme.txt file in the SourceDir folder with instructions + # for launching the application + ``` + +3. Add necessary assets: + - Copy license file: `copy assets\openms_license.rtf SourceDir\` + - Copy app icon: `copy assets\openms.ico SourceDir\` + - Create success message script: + ```vbscript + ' ShowSuccessMessage.vbs + MsgBox "The " & "%APP_NAME%" & " application is successfully installed.", vbInformation, "Installation Complete" + ``` + +### 4. Generate WiX Source Files + +1. Generate component list from your files: + + ```batch + wix\heat.exe dir SourceDir -gg -sfrag -sreg -srd -template component -cg StreamlitExeFiles -dr AppSubFolder -out streamlit_exe_files.wxs + ``` + +2. Create main WiX configuration file (streamlit_exe.wxs): + + ```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + NOT Installed + + + + + + + ``` + +### 5. Build the MSI + +1. Compile WiX source files: + + ```batch + # Generate wixobj files from the WiX source files + wix\candle.exe streamlit_exe.wxs streamlit_exe_files.wxs + ``` + +2. Link and create MSI: + ```batch + # Create the MSI installer from the wixobj files + # The -sice:ICE60 flag stops a warning about duplicate component GUIDs, which can happen when heat.exe auto-generates components + wix\light.exe -ext WixUIExtension -sice:ICE60 -o %APP_NAME%.msi streamlit_exe_files.wixobj streamlit_exe.wixobj + ``` + +### 6. Additional Notes + +- The generated MSI will create desktop and start menu shortcuts +- Installation requires elevated privileges +- A success message will be shown after installation +- The installer includes a proper license agreement page +- All files will be installed in Program Files by default + +For more detailed customization options, refer to the [WiX Toolset documentation](https://wixtoolset.org/documentation/). + +:warning: The `APP_UpgradeCode` GUID should be unique for your application. Generate a new one if you're creating a different app. diff --git a/docs/win_exe_with_pyinstaller.md b/docs/win_exe_with_pyinstaller.md new file mode 100644 index 0000000..7112691 --- /dev/null +++ b/docs/win_exe_with_pyinstaller.md @@ -0,0 +1,140 @@ +## πŸ’» Create a window executable of streamlit app with pyinstaller +:heavy_check_mark: +Tested with streamlit v1.29.0, python v3.11.4 + +:warning: Support until streamlit version `1.29.0` +:point_right: For higher version, try streamlit app with embeddable python #TODO add link + +To create an executable for Streamlit app on Windows, we'll use an pyinstaller. +Here's a step-by-step guide: + +### virtual environment + +``` +# create an environment +python -m venv + +# activate an environment +.\myenv\Scripts\Activate.bat + +# install require packages +pip install -r requirements.txt + +#install pyinstaller +pip install pyinstaller +``` + +### streamlit files + +create a run_app.py and add this lines of codes +``` +from streamlit.web import cli + +if __name__=='__main__': + cli._main_run_clExplicit( + file="app.py", command_line="streamlit run" + ) + # we will create this function inside our streamlit framework + +``` + +### write function in cli.py + +Now, navigate to the inside streamlit environment + +here you go + +``` +\Lib\site-packages\streamlit\web\cli.py +``` +for using our virtual environment, add this magic function to cli.py file: +``` +#can be modify name as given in run_app.py +#use underscore at beginning +def _main_run_clExplicit(file, command_line, args=[], flag_options=[]): + main._is_running_with_streamlit = True + bootstrap.run(file, command_line, args, flag_options) +``` + +### Hook folder +Now, need to hook to get streamlit metadata +organized as folder, where the pycache infos will save +like: \hooks\hook-streamlit.py + +``` +from PyInstaller.utils.hooks import copy_metadata +datas = [] +datas += copy_metadata('streamlit') +datas += copy_metadata('pyopenms') +# can add new package e-g +datas += copy_metadata('captcha') +``` + +### compile the app +Now, ready for compilation +``` +pyinstaller --onefile --additional-hooks-dir ./hooks run_app.py --clean + +#--onefile create join binary file ?? +#will create run_app.spec file +#--clean delete cache and removed temporary files before building +#--additional-hooks-dir path to search for hook +``` + +### streamlit config +To access streamlit config create file in root +(or just can be in output folder) +.streamlit\config.toml + +``` +# content of .streamlit\config.toml +[global] +developmentMode = false + +[server] +port = 8502 +``` + +### copy necessary files to dist folder +``` +cp -r .streamlit dist/.streamlit +cp -r pages dist/pages +cp -r src dist/src +cp -r assets dist/assets +cp app.py dist/ +cp presets.json dist/ +``` + +Remove the server address from the bundled config so Streamlit uses `localhost` (default) instead of `0.0.0.0`, which doesn't work as a connect address on Windows: + +```powershell +(Get-Content dist/.streamlit/config.toml) -notmatch '^address' | Set-Content dist/.streamlit/config.toml +``` + + +### add datas in run_app.spec (.spec file) +Add DATAS to the run_app.spec just created by compilation + +``` +datas=[ + ("myenv/Lib/site-packages/altair/vegalite/v4/schema/vega-lite-schema.json","./altair/vegalite/v4/schema/"), + ("myenv/Lib/site-packages/streamlit/static", "./streamlit/static"), + ("myenv/Lib/site-packages/streamlit/runtime", "./streamlit/runtime"), + ("myenv/Lib/site-packages/pyopenms", "./pyopenms/"), + # Add new datas e-g we add in hook captcha + ("myenv/Lib/site-packages/captcha", "./captcha/") + ] +``` +### run final step to make executable +All the modifications in datas should be loaded with +``` +pyinstaller run_app.spec --clean +``` +#### πŸš€ After successfully completing all these steps, the Windows executable will be available in the dist folder. + +:pencil: you can still change the configuration of streamlit app with .streamlit/config.toml file e-g provide different port, change upload size etc + +ℹ️ if problem with altair, Try version altair==4.0.1, and again compile + +## Build executable in github action automatically +Automate the process of building executables for your project with the GitHub action example [Test streamlit executable for Windows with pyinstaller](https://github.com/OpenMS/streamlit-template/blob/main/.github/workflows/test-win-exe-w-pyinstaller.yaml) diff --git a/gdpr_consent/dist/bundle.js b/gdpr_consent/dist/bundle.js index 8614457..0a48bfd 100644 --- a/gdpr_consent/dist/bundle.js +++ b/gdpr_consent/dist/bundle.js @@ -235,7 +235,7 @@ eval("__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpac /***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => { "use strict"; -eval("__webpack_require__.r(__webpack_exports__);\n/* harmony import */ var streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! streamlit-component-lib */ \"./node_modules/streamlit-component-lib/dist/index.js\");\nvar __awaiter = (undefined && undefined.__awaiter) || function (thisArg, _arguments, P, generator) {\n function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }\n return new (P || (P = Promise))(function (resolve, reject) {\n function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }\n function rejected(value) { try { step(generator[\"throw\"](value)); } catch (e) { reject(e); } }\n function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }\n step((generator = generator.apply(thisArg, _arguments || [])).next());\n });\n};\nvar __generator = (undefined && undefined.__generator) || function (thisArg, body) {\n var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;\n return g = { next: verb(0), \"throw\": verb(1), \"return\": verb(2) }, typeof Symbol === \"function\" && (g[Symbol.iterator] = function() { return this; }), g;\n function verb(n) { return function (v) { return step([n, v]); }; }\n function step(op) {\n if (f) throw new TypeError(\"Generator is already executing.\");\n while (g && (g = 0, op[0] && (_ = 0)), _) try {\n if (f = 1, y && (t = op[0] & 2 ? y[\"return\"] : op[0] ? y[\"throw\"] || ((t = y[\"return\"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;\n if (y = 0, t) op = [op[0] & 2, t.value];\n switch (op[0]) {\n case 0: case 1: t = op; break;\n case 4: _.label++; return { value: op[1], done: false };\n case 5: _.label++; y = op[1]; op = [0]; continue;\n case 7: op = _.ops.pop(); _.trys.pop(); continue;\n default:\n if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }\n if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }\n if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }\n if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }\n if (t[2]) _.ops.pop();\n _.trys.pop(); continue;\n }\n op = body.call(thisArg, _);\n } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }\n if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };\n }\n};\n\n// Defines the configuration for Klaro\nvar klaroConfig = {\n mustConsent: true,\n acceptAll: true,\n services: []\n};\n// This will make klaroConfig globally accessible\nwindow.klaroConfig = klaroConfig;\n// Function to safely access the Klaro manager\nfunction getKlaroManager() {\n var _a;\n return ((_a = window.klaro) === null || _a === void 0 ? void 0 : _a.getManager) ? window.klaro.getManager() : null;\n}\n// Waits until Klaro Manager is available\nfunction waitForKlaroManager() {\n return __awaiter(this, arguments, void 0, function (maxWaitTime, interval) {\n var startTime, klaroManager;\n if (maxWaitTime === void 0) { maxWaitTime = 5000; }\n if (interval === void 0) { interval = 100; }\n return __generator(this, function (_a) {\n switch (_a.label) {\n case 0:\n startTime = Date.now();\n _a.label = 1;\n case 1:\n if (!(Date.now() - startTime < maxWaitTime)) return [3 /*break*/, 3];\n klaroManager = getKlaroManager();\n if (klaroManager) {\n return [2 /*return*/, klaroManager];\n }\n return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, interval); })];\n case 2:\n _a.sent();\n return [3 /*break*/, 1];\n case 3: throw new Error(\"Klaro manager did not become available within the allowed time.\");\n }\n });\n });\n}\n// Helper function to handle unknown errors\nfunction handleError(error) {\n if (error instanceof Error) {\n console.error(\"Error:\", error.message);\n }\n else {\n console.error(\"Unknown error:\", error);\n }\n}\n// Tracking was accepted\nfunction callback() {\n return __awaiter(this, void 0, void 0, function () {\n var manager, return_vals, _i, _a, service, error_1;\n return __generator(this, function (_b) {\n switch (_b.label) {\n case 0:\n _b.trys.push([0, 2, , 3]);\n return [4 /*yield*/, waitForKlaroManager()];\n case 1:\n manager = _b.sent();\n if (manager.confirmed) {\n return_vals = {};\n for (_i = 0, _a = klaroConfig.services; _i < _a.length; _i++) {\n service = _a[_i];\n return_vals[service.name] = manager.getConsent(service.name);\n }\n streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentValue(return_vals);\n }\n return [3 /*break*/, 3];\n case 2:\n error_1 = _b.sent();\n handleError(error_1);\n return [3 /*break*/, 3];\n case 3: return [2 /*return*/];\n }\n });\n });\n}\n// Stores if the component has been rendered before\nvar rendered = false;\nfunction onRender(event) {\n // Klaro does not work if embedded multiple times\n if (rendered) {\n return;\n }\n rendered = true;\n var data = event.detail;\n if (data.args['google_analytics']) {\n klaroConfig.services.push({\n name: 'google-analytics',\n cookies: [\n /^_ga(_.*)?/ // we delete the Google Analytics cookies if the user declines its use\n ],\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['piwik_pro']) {\n klaroConfig.services.push({\n name: 'piwik-pro',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['matomo']) {\n klaroConfig.services.push({\n name: 'matomo',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n // Create a new script element\n var script = document.createElement('script');\n // Set the necessary attributes\n script.defer = true;\n script.type = 'application/javascript';\n script.src = 'https://cdn.kiprotect.com/klaro/v0.7/klaro.js';\n // Set the klaro config\n script.setAttribute('data-config', 'klaroConfig');\n // Append the script to the head or body\n document.head.appendChild(script);\n}\n// Attach our `onRender` handler to Streamlit's render event.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.events.addEventListener(streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.RENDER_EVENT, onRender);\n// Tell Streamlit we're ready to start receiving data. We won't get our\n// first RENDER_EVENT until we call this function.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentReady();\n// Finally, tell Streamlit to update the initial height.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setFrameHeight(1000);\n\n\n//# sourceURL=webpack://gdpr_consent/./src/main.ts?"); +eval("__webpack_require__.r(__webpack_exports__);\n/* harmony import */ var streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! streamlit-component-lib */ \"./node_modules/streamlit-component-lib/dist/index.js\");\nvar __awaiter = (undefined && undefined.__awaiter) || function (thisArg, _arguments, P, generator) {\n function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }\n return new (P || (P = Promise))(function (resolve, reject) {\n function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }\n function rejected(value) { try { step(generator[\"throw\"](value)); } catch (e) { reject(e); } }\n function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }\n step((generator = generator.apply(thisArg, _arguments || [])).next());\n });\n};\nvar __generator = (undefined && undefined.__generator) || function (thisArg, body) {\n var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;\n return g = { next: verb(0), \"throw\": verb(1), \"return\": verb(2) }, typeof Symbol === \"function\" && (g[Symbol.iterator] = function() { return this; }), g;\n function verb(n) { return function (v) { return step([n, v]); }; }\n function step(op) {\n if (f) throw new TypeError(\"Generator is already executing.\");\n while (g && (g = 0, op[0] && (_ = 0)), _) try {\n if (f = 1, y && (t = op[0] & 2 ? y[\"return\"] : op[0] ? y[\"throw\"] || ((t = y[\"return\"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;\n if (y = 0, t) op = [op[0] & 2, t.value];\n switch (op[0]) {\n case 0: case 1: t = op; break;\n case 4: _.label++; return { value: op[1], done: false };\n case 5: _.label++; y = op[1]; op = [0]; continue;\n case 7: op = _.ops.pop(); _.trys.pop(); continue;\n default:\n if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }\n if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }\n if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }\n if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }\n if (t[2]) _.ops.pop();\n _.trys.pop(); continue;\n }\n op = body.call(thisArg, _);\n } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }\n if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };\n }\n};\n\n// Defines the configuration for Klaro\nvar klaroConfig = {\n mustConsent: true,\n acceptAll: true,\n services: []\n};\n// This will make klaroConfig globally accessible\nwindow.klaroConfig = klaroConfig;\n// Function to safely access the Klaro manager\nfunction getKlaroManager() {\n var _a;\n return ((_a = window.klaro) === null || _a === void 0 ? void 0 : _a.getManager) ? window.klaro.getManager() : null;\n}\n// Waits until Klaro Manager is available\nfunction waitForKlaroManager() {\n return __awaiter(this, arguments, void 0, function (maxWaitTime, interval) {\n var startTime, klaroManager;\n if (maxWaitTime === void 0) { maxWaitTime = 5000; }\n if (interval === void 0) { interval = 100; }\n return __generator(this, function (_a) {\n switch (_a.label) {\n case 0:\n startTime = Date.now();\n _a.label = 1;\n case 1:\n if (!(Date.now() - startTime < maxWaitTime)) return [3 /*break*/, 3];\n klaroManager = getKlaroManager();\n if (klaroManager) {\n return [2 /*return*/, klaroManager];\n }\n return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, interval); })];\n case 2:\n _a.sent();\n return [3 /*break*/, 1];\n case 3: throw new Error(\"Klaro manager did not become available within the allowed time.\");\n }\n });\n });\n}\n// Helper function to handle unknown errors\nfunction handleError(error) {\n if (error instanceof Error) {\n console.error(\"Error:\", error.message);\n }\n else {\n console.error(\"Unknown error:\", error);\n }\n}\n// Tracking was accepted\nfunction callback() {\n return __awaiter(this, void 0, void 0, function () {\n var manager, return_vals, _i, _a, service, error_1;\n return __generator(this, function (_b) {\n switch (_b.label) {\n case 0:\n _b.trys.push([0, 2, , 3]);\n return [4 /*yield*/, waitForKlaroManager()];\n case 1:\n manager = _b.sent();\n if (manager.confirmed) {\n return_vals = {};\n for (_i = 0, _a = klaroConfig.services; _i < _a.length; _i++) {\n service = _a[_i];\n return_vals[service.name] = manager.getConsent(service.name);\n }\n streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentValue(return_vals);\n }\n return [3 /*break*/, 3];\n case 2:\n error_1 = _b.sent();\n handleError(error_1);\n return [3 /*break*/, 3];\n case 3: return [2 /*return*/];\n }\n });\n });\n}\n// Stores if the component has been rendered before\nvar rendered = false;\nfunction onRender(event) {\n // Klaro does not work if embedded multiple times\n if (rendered) {\n return;\n }\n rendered = true;\n var data = event.detail;\n if (data.args['google_analytics']) {\n klaroConfig.services.push({\n name: 'google-analytics',\n cookies: [\n /^_ga(_.*)?/ // we delete the Google Analytics cookies if the user declines its use\n ],\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['piwik_pro']) {\n klaroConfig.services.push({\n name: 'piwik-pro',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['matomo']) {\n klaroConfig.services.push({\n name: 'matomo',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n // Link the consent banner to the privacy policy. Setting privacyPolicyUrl\n // on the 'zz' fallback language makes Klaro render its default\n // \"To learn more, please read our privacy policy.\" text with the URL,\n // regardless of the browser locale.\n if (data.args['privacy_policy']) {\n klaroConfig.translations = {\n zz: {\n privacyPolicyUrl: data.args['privacy_policy']\n }\n };\n }\n // Create a new script element\n var script = document.createElement('script');\n // Set the necessary attributes\n script.defer = true;\n script.type = 'application/javascript';\n script.src = 'https://cdn.kiprotect.com/klaro/v0.7/klaro.js';\n // Set the klaro config\n script.setAttribute('data-config', 'klaroConfig');\n // Append the script to the head or body\n document.head.appendChild(script);\n}\n// Attach our `onRender` handler to Streamlit's render event.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.events.addEventListener(streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.RENDER_EVENT, onRender);\n// Tell Streamlit we're ready to start receiving data. We won't get our\n// first RENDER_EVENT until we call this function.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentReady();\n// Finally, tell Streamlit to update the initial height.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setFrameHeight(1000);\n\n\n//# sourceURL=webpack://gdpr_consent/./src/main.ts?"); /***/ }), diff --git a/gdpr_consent/src/main.ts b/gdpr_consent/src/main.ts index 059fef8..408f4a4 100644 --- a/gdpr_consent/src/main.ts +++ b/gdpr_consent/src/main.ts @@ -14,6 +14,7 @@ let klaroConfig: { mustConsent: boolean; acceptAll: boolean; services: Service[]; + translations?: Record; } = { mustConsent: true, acceptAll: true, @@ -125,6 +126,18 @@ function onRender(event: Event): void { ) } + // Link the consent banner to the privacy policy. Setting privacyPolicyUrl + // on the 'zz' fallback language makes Klaro render its default + // "To learn more, please read our privacy policy." text with the URL, + // regardless of the browser locale. + if (data.args['privacy_policy']) { + klaroConfig.translations = { + zz: { + privacyPolicyUrl: data.args['privacy_policy'] + } + } + } + // Create a new script element var script = document.createElement('script') diff --git a/requirements.txt b/requirements.txt index 5d20693..0e26851 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,9 +16,9 @@ cachetools==5.5.2 # via streamlit captcha==0.7.1 # via src (pyproject.toml) -certifi==2025.1.31 +certifi==2025.8.3 # via requests -charset-normalizer==3.4.1 +charset-normalizer==3.4.3 # via requests click==8.1.8 # via streamlit @@ -26,11 +26,11 @@ contourpy==1.3.1 # via matplotlib cycler==0.12.1 # via matplotlib -fonttools==4.56.0 +fonttools==4.59.2 # via matplotlib gitdb==4.0.12 # via gitpython -gitpython==3.1.44 +gitpython==3.1.45 # via streamlit idna==3.10 # via requests @@ -48,7 +48,7 @@ markupsafe==3.0.2 # via jinja2 matplotlib==3.10.1 # via pyopenms -narwhals==1.32.0 +narwhals==2.5.0 # via altair numpy==1.26.4 # via @@ -59,7 +59,7 @@ numpy==1.26.4 # pyopenms # src (pyproject.toml) # streamlit -packaging==24.2 +packaging==25.0 # via # altair # matplotlib @@ -77,7 +77,7 @@ pillow==11.1.0 # streamlit plotly==5.22.0 # via src (pyproject.toml) -protobuf==5.29.4 +protobuf==6.32.0 # via streamlit psutil==7.0.0 # via src (pyproject.toml) @@ -85,7 +85,7 @@ pyarrow==19.0.1 # via streamlit pydeck==0.9.1 # via streamlit -pyopenms>=3.0.0,<3.5 +pyopenms==3.5.0 # via src (pyproject.toml) pyopenms-viz==1.0.0 # via src (pyproject.toml) @@ -111,7 +111,7 @@ six==1.17.0 # via python-dateutil smmap==5.0.2 # via gitdb -streamlit==1.43.0 +streamlit==1.49.1 # via # src (pyproject.toml) # streamlit-js-eval @@ -151,8 +151,8 @@ cython easypqp>=0.1.34 pyprophet>=2.2.0 mygene +statsmodels + # Redis Queue dependencies (for online mode) redis>=5.0.0 rq>=1.16.0 -statsmodels -polars \ No newline at end of file diff --git a/src/common/admin.py b/src/common/admin.py index 2408c0a..a4414b9 100644 --- a/src/common/admin.py +++ b/src/common/admin.py @@ -9,6 +9,7 @@ from pathlib import Path import streamlit as st +from streamlit.errors import StreamlitSecretNotFoundError def is_admin_configured() -> bool: @@ -20,7 +21,7 @@ def is_admin_configured() -> bool: """ try: return bool(st.secrets.get("admin", {}).get("password")) - except (FileNotFoundError, KeyError): + except (FileNotFoundError, KeyError, StreamlitSecretNotFoundError): return False except Exception: return False diff --git a/src/common/captcha_.py b/src/common/captcha_.py index 498b133..282e124 100644 --- a/src/common/captcha_.py +++ b/src/common/captcha_.py @@ -186,7 +186,7 @@ def add_page(main_script_path_str: str, page_name: str) -> None: # define the function for the captcha control -def captcha_control(): +def captcha_control(privacy_policy_url: str = ""): """ Control and verification of a CAPTCHA to ensure the user is not a robot. @@ -199,6 +199,10 @@ def captcha_control(): The CAPTCHA text is generated as a session state and should not change during refreshes. + Args: + privacy_policy_url (str, optional): URL shown as the privacy policy link + in the GDPR consent banner. Defaults to "". + Returns: None """ @@ -214,7 +218,10 @@ def captcha_control(): with st.spinner(): # Ask for consent st.session_state.tracking_consent = consent_component( - google_analytics=ga, piwik_pro=pp, matomo=mt + google_analytics=ga, + piwik_pro=pp, + matomo=mt, + privacy_policy=privacy_policy_url, ) if st.session_state.tracking_consent is None: # No response by user yet diff --git a/src/common/common.py b/src/common/common.py index 643a224..c048064 100644 --- a/src/common/common.py +++ b/src/common/common.py @@ -31,6 +31,37 @@ # Detect system platform OS_PLATFORM = sys.platform +# Default legal/GDPR page links. These point to the centrally maintained +# official OpenMS pages. Forks that self-host should override them via the +# "legal_links" key in settings.json (an Impressum must name the actual +# operator). The defaults live here too β€” not only in settings.json β€” so that +# downstream apps built from an older settings.json without a "legal_links" +# key still inherit working legal links by default. +DEFAULT_LEGAL_LINKS = { + "impressum": "https://openms.de/impressum", + "privacy": "https://openms.de/privacy", + "terms": "https://openms.de/terms", +} + + +def get_legal_links() -> dict[str, str]: + """ + Return the legal page URLs (Impressum, Privacy Policy, Terms of Use). + + Values from the "legal_links" object in settings.json override the + built-in OpenMS defaults. Empty override values are ignored so a blank + entry can't erase a default. + + Returns: + dict[str, str]: Mapping of "impressum", "privacy" and "terms" to URLs. + """ + overrides = ( + st.session_state.settings.get("legal_links", {}) + if "settings" in st.session_state + else {} + ) + return {**DEFAULT_LEGAL_LINKS, **{k: v for k, v in overrides.items() if v}} + def is_safe_workspace_name(name: str) -> bool: """ @@ -519,7 +550,7 @@ def page_setup(page: str = "") -> dict[str, Any]: # Render the sidebar params = render_sidebar(page) - captcha_control() + captcha_control(privacy_policy_url=get_legal_links()["privacy"]) # If run in hosted mode, show captcha as long as it has not been solved # if not "local" in sys.argv: @@ -532,7 +563,7 @@ def page_setup(page: str = "") -> dict[str, Any]: "controllo" in params.keys() and params["controllo"] == False ): # Apply captcha by calling the captcha_control function - captcha_control() + captcha_control(privacy_policy_url=get_legal_links()["privacy"]) return params @@ -764,6 +795,19 @@ def change_workspace(): f'
{app_name}
Version: {version_info}
', unsafe_allow_html=True, ) + + # Legal links (Impressum, Privacy Policy, Terms of Use), shown on every + # page. URLs are configurable via "legal_links" in settings.json. + links = get_legal_links() + st.markdown( + '
' + f'Impressum · ' + f'Privacy Policy · ' + f'Terms of Use' + "
", + unsafe_allow_html=True, + ) return params diff --git a/src/mzmlfileworkflow.py b/src/mzmlfileworkflow.py new file mode 100644 index 0000000..94faa2f --- /dev/null +++ b/src/mzmlfileworkflow.py @@ -0,0 +1,107 @@ +import streamlit as st +from pathlib import Path +import pyopenms as poms +import pandas as pd +import time +from datetime import datetime +from src.common.common import reset_directory, show_fig, show_table +import plotly.express as px + + +def mzML_file_get_num_spectra(filepath): + """ + Load an mzML file, retrieve the number of spectra, and return it. + + This function loads an mzML file specified by `filepath` and extracts the number of spectra + contained within the file using the OpenMS library. It temporarily pauses for 2 seconds to + simulate a heavy task before retrieving the number of spectra. + + Args: + filepath (str): The path to the mzML file to be loaded and analyzed. + + Returns: + int: The number of spectra present in the mzML file. + """ + exp = poms.MSExperiment() + poms.MzMLFile().load(filepath, exp) + time.sleep(2) + return exp.size() + + +def run_workflow(params, result_dir): + """Load each mzML file into pyOpenMS Experiment and get the number of spectra.""" + + result_dir = Path(result_dir, datetime.now().strftime("%Y-%m-%d %H_%M_%S")) + # delete old workflow results and set new directory + reset_directory(result_dir) + + # collect spectra numbers + num_spectra = [] + + # use st.status to print info while running the workflow + with st.status( + "Loading mzML files and getting number of spectra...", expanded=True + ) as status: + # get selected mzML files from parameters + for file in params["example-workflow-selected-mzML-files"]: + # logging file name in status + st.write(f"Reading mzML file: {file} ...") + + # reading mzML file, getting num spectra and adding some extra time + num_spectra.append( + mzML_file_get_num_spectra( + str( + Path( + st.session_state["workspace"], "mzML-files", file + ".mzML" + ) + ) + ) + ) + + # set status as complete and collapse box + status.update(label="Complete!", expanded=False) + + # create and save result dataframe + df = pd.DataFrame( + { + "filenames": params["example-workflow-selected-mzML-files"], + "number of spectra": num_spectra, + } + ) + df.to_csv(Path(result_dir, "result.tsv"), sep="\t", index=False) + +@st.fragment +def result_section(result_dir): + if not Path(result_dir).exists(): + st.error("No results to show yet. Please run a workflow first!") + return + + date_strings = [f.name for f in Path(result_dir).iterdir() if f.is_dir()] + + result_dirs = sorted(date_strings, key=lambda date: datetime.strptime(date, "%Y-%m-%d %H_%M_%S"))[::-1] + + run_dir = st.selectbox("select result from run", result_dirs) + + if run_dir is None: + st.error("Please select a result from a run!") + return + + result_dir = Path(result_dir, run_dir) + # visualize workflow results if there are any + result_file_path = Path(result_dir, "result.tsv") + + if result_file_path.exists(): + df = pd.read_csv(result_file_path, sep="\t", index_col="filenames") + + if not df.empty: + tabs = st.tabs(["πŸ“ data", "πŸ“Š plot"]) + + with tabs[0]: + show_table(df, "mzML-workflow-result") + + with tabs[1]: + fig = px.bar(df) + st.info( + "πŸ’‘ Download figure with camera icon in top right corner. File format can be specified in settings." + ) + show_fig(fig, "mzML-workflow-results") \ No newline at end of file diff --git a/src/peptide_mz_calculator.py b/src/peptide_mz_calculator.py new file mode 100644 index 0000000..5b75c57 --- /dev/null +++ b/src/peptide_mz_calculator.py @@ -0,0 +1,107 @@ +""" +Peptide M/Z Calculator Backend + +This module provides backend functions for peptide mass spectrometry calculations +using pyOpenMS AASequence.fromString() directly with minimal parsing overhead. +""" + +from typing import Dict, Any, Tuple +import pyopenms as poms + + +def calculate_peptide_mz(sequence: str, charge_state: int) -> Dict[str, Any]: + """Calculate m/z ratio for a peptide using AASequence.fromString() directly. + + Args: + sequence: Peptide sequence string (AASequence.fromString() compatible) + charge_state: Charge state for m/z calculation + + Returns: + Dictionary with calculation results + + Raises: + ValueError: If sequence is invalid or charge state is invalid + """ + sequence = sequence.strip() + if not sequence: + raise ValueError("Peptide sequence cannot be empty") + + if charge_state < 1: + raise ValueError("Charge state must be a positive integer") + + try: + # Use AASequence.fromString() directly - it supports many formats natively + aa_sequence = poms.AASequence.fromString(sequence) + except Exception as e: + raise ValueError(f"Invalid sequence format: {str(e)}") from e + + # Calculate properties + mz_ratio = aa_sequence.getMZ(charge_state) + mono_weight = aa_sequence.getMonoWeight() + formula = aa_sequence.getFormula() + + # Extract clean amino acid sequence for composition + unmodified_aa_sequence = aa_sequence.toUnmodifiedString() + + # Calculate amino acid composition + aa_composition = {} + for aa in unmodified_aa_sequence: + aa_composition[aa] = aa_composition.get(aa, 0) + 1 + + return { + "mz_ratio": mz_ratio, + "monoisotopic_mass": mono_weight, + "molecular_formula": formula.toString(), + "charge_state": charge_state, + "sequence_length": len(unmodified_aa_sequence), + "aa_composition": aa_composition, + "success": True, + } + +def calculate_peptide_mz_range( + sequence: str, + charge_range: Tuple[int, int] +) -> Dict[str, Any]: + """Calculate m/z ratios for multiple charge states. + + Args: + sequence: Peptide sequence string + charge_range: Tuple of (min_charge, max_charge) inclusive + + Returns: + Dictionary containing results for all charge states + """ + min_charge, max_charge = charge_range + charge_results = {} + + # Calculate for each charge state + for charge in range(min_charge, max_charge + 1): + result = calculate_peptide_mz(sequence, charge) + charge_results[charge] = result + + # Use first result as base and add charge_results + base_result = charge_results[min_charge] + return { + **base_result, + "charge_results": charge_results, + "charge_range": charge_range, + } + + +def validate_sequence(sequence: str) -> Tuple[bool, str]: + """Validate if sequence can be parsed by AASequence.fromString(). + + Args: + sequence: Sequence string to validate + + Returns: + Tuple of (is_valid, error_message) + """ + if not sequence.strip(): + return False, "Sequence cannot be empty" + + try: + poms.AASequence.fromString(sequence.strip()) + return True, "" + except Exception as e: + return False, f"Invalid sequence format: {str(e)}" diff --git a/src/python-tools/example.py b/src/python-tools/example.py new file mode 100644 index 0000000..50a7b47 --- /dev/null +++ b/src/python-tools/example.py @@ -0,0 +1,67 @@ +import json +import sys + +############################ +# default paramter values # +########################### +# +# Mandatory keys for each parameter +# key: a unique identifier +# value: the default value +# +# Optional keys for each parameter +# name: the name of the parameter +# hide: don't show the parameter in the parameter section (e.g. for input/output files) +# options: a list of valid options for the parameter +# min: the minimum value for the parameter (int and float) +# max: the maximum value for the parameter (int and float) +# step_size: the step size for the parameter (int and float) +# help: a description of the parameter +# widget_type: the type of widget to use for the parameter (default: auto) +# advanced: whether or not the parameter is advanced (default: False) + +DEFAULTS = [ + {"key": "in", "value": [], "help": "Input files for Python Script.", "hide": True}, + {"key": "out", "value": [], "help": "Output files for Python Script.", "hide": True}, + { + "key": "number-slider", + "name": "number of features", + "value": 6, + "min": 2, + "max": 10, + "help": "How many features to consider.", + "widget_type": "slider", + "step_size": 2, + }, + { + "key": "selectbox-example", + "name": "select something", + "value": "a", + "options": ["a", "b", "c"], + }, + { + "key": "adavanced-input", + "name": "advanced parameter", + "value": 5, + "step_size": 5, + "help": "An advanced example parameter.", + "advanced": True, + }, + { + "key": "checkbox", "value": True, "name": "boolean" + } +] + +def get_params(): + if len(sys.argv) > 1: + with open(sys.argv[1], "r") as f: + return json.load(f) + else: + return {} + +if __name__ == "__main__": + params = get_params() + # Add code here: + print("Writing stdout which will get logged...") + print("Parameters for this example Python tool:") + print(json.dumps(params, indent=4)) \ No newline at end of file diff --git a/src/python-tools/export_consensus_feature_df.py b/src/python-tools/export_consensus_feature_df.py new file mode 100644 index 0000000..9f0ceb1 --- /dev/null +++ b/src/python-tools/export_consensus_feature_df.py @@ -0,0 +1,46 @@ +import json +import sys +from pyopenms import ConsensusXMLFile, ConsensusMap +from pathlib import Path + +############################ +# default paramter values # +########################### +# +# Mandatory keys for each parameter +# key: a unique identifier +# value: the default value +# +# Optional keys for each parameter +# name: the name of the parameter +# hide: don't show the parameter in the parameter section (e.g. for input/output files) +# options: a list of valid options for the parameter +# min: the minimum value for the parameter (int and float) +# max: the maximum value for the parameter (int and float) +# step_size: the step size for the parameter (int and float) +# help: a description of the parameter +# widget_type: the type of widget to use for the parameter (default: auto) +# advanced: whether or not the parameter is advanced (default: False) + +DEFAULTS = [ + {"key": "in", "value": "", "help": "Input consensusXML file.", "hide": True}, +] + +def get_params(): + if len(sys.argv) > 1: + with open(sys.argv[1], "r") as f: + return json.load(f) + else: + return {} + +if __name__ == "__main__": + params = get_params() + # Add code here: + cm = ConsensusMap() + ConsensusXMLFile().load(params["in"], cm) + df = cm.get_df() + df = df.rename(columns={col: Path(col).name for col in df.columns}) + df = df.reset_index() + df = df.drop(columns=["id", "sequence"]) + df.insert(0, "metabolite", df.apply(lambda x: f"{round(x['mz'], 4)}@{round(x['rt'], 2)}", axis=1)) + df.to_csv(Path(params["in"]).with_suffix(".tsv"), sep="\t", index=False) \ No newline at end of file diff --git a/src/run_subprocess.py b/src/run_subprocess.py new file mode 100644 index 0000000..a5f25df --- /dev/null +++ b/src/run_subprocess.py @@ -0,0 +1,57 @@ +import streamlit as st +import subprocess + + +def run_subprocess(args: list[str], result_dict: dict) -> None: + """ + Run a subprocess and capture its output. + + Args: + args (list[str]): The command and its arguments as a list of strings. + variables (list[str]): Additional variables needed for the subprocess (not used in this code). + result_dict dict: A dictionary to store the success status (bool) and the captured log (str). + + Returns: + None + """ + + # Run the subprocess and capture its output + process = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True + ) + + # Lists to store the captured standard output and standard error + stdout_ = [] + stderr_ = [] + + # Capture the standard output of the subprocess + while True: + output = process.stdout.readline() + if output == "" and process.poll() is not None: + break + if output: + # Print every line of standard output on the Streamlit page + st.text(output.strip()) + # Append the line to store in the log + stdout_.append(output.strip()) + + # Capture the standard error of the subprocess + while True: + error = process.stderr.readline() + if error == "" and process.poll() is not None: + break + if error: + # Print every line of standard error on the Streamlit page, marking it as an error + st.error(error.strip()) + # Append the line to store in the log of errors + stderr_.append(error.strip()) + + # Check if the subprocess ran successfully (return code 0) + if process.returncode == 0: + result_dict["success"] = True + # Save all lines from standard output to the log + result_dict["log"] = " ".join(stdout_) + else: + result_dict["success"] = False + # Save all lines from standard error to the log, even if the process encountered an error + result_dict["log"] = " ".join(stderr_) diff --git a/src/simpleworkflow.py b/src/simpleworkflow.py new file mode 100644 index 0000000..0eccb9e --- /dev/null +++ b/src/simpleworkflow.py @@ -0,0 +1,13 @@ +import time + +import numpy as np +import pandas as pd +import streamlit as st + + +@st.cache_data +def generate_random_table(x, y): + """Example for a cached table""" + df = pd.DataFrame(np.random.randn(x, y)) + time.sleep(2) + return df diff --git a/src/view.py b/src/view.py new file mode 100644 index 0000000..3582b1a --- /dev/null +++ b/src/view.py @@ -0,0 +1,327 @@ +import numpy as np +import pandas as pd +from pathlib import Path +import plotly.express as px +import plotly.graph_objects as go +import streamlit as st +import pyopenms as poms +from src.common.common import show_fig, display_large_dataframe +from typing import Union + + +def get_df(file: Union[str, Path]) -> pd.DataFrame: + """ + Load a Mass Spectrometry (MS) experiment from a given mzML file and return + a pandas dataframe representation of the experiment. + + Args: + file (Union[str, Path]): The path to the mzML file to load. + + Returns: + pd.DataFrame: A pandas DataFrame with the following columns: "mslevel", + "precursormz", "mzarray", and "intarray". The "mzarray" and "intarray" + columns contain NumPy arrays with the m/z and intensity values for each + spectrum in the mzML file, respectively. + """ + exp = poms.MSExperiment() + poms.MzMLFile().load(str(file), exp) + df_spectra = exp.get_df() + df_spectra.rename(columns={ + 'rt': 'RT', + 'ms_level': 'MS level', + 'mz_array': 'mzarray', + 'intensity_array': 'intarray', + }, inplace=True) + precs = [] + for spec in exp: + p = spec.getPrecursors() + if p: + precs.append(p[0].getMZ()) + else: + precs.append(np.nan) + df_spectra["precursor m/z"] = precs + + # Drop spectra without peaks + df_spectra = df_spectra[df_spectra["mzarray"].apply(lambda x: len(x) > 0)] + + df_spectra["max intensity m/z"] = df_spectra.apply( + lambda x: x["mzarray"][x["intarray"].argmax()], axis=1 + ) + if not df_spectra.empty: + st.session_state["view_spectra"] = df_spectra + else: + st.session_state["view_spectra"] = pd.DataFrame() + exp_ms2 = poms.MSExperiment() + exp_ms1 = poms.MSExperiment() + for spec in exp: + if spec.getMSLevel() == 1: + exp_ms1.addSpectrum(spec) + elif spec.getMSLevel() == 2: + exp_ms2.addSpectrum(spec) + _long_rename = {'rt': 'RT', 'intensity': 'inty'} + if not exp_ms1.empty(): + st.session_state["view_ms1"] = exp_ms1.get_df(long=True).rename(columns=_long_rename) + else: + st.session_state["view_ms1"] = pd.DataFrame(columns=['RT', 'mz', 'inty']) + if not exp_ms2.empty(): + st.session_state["view_ms2"] = exp_ms2.get_df(long=True).rename(columns=_long_rename) + else: + st.session_state["view_ms2"] = pd.DataFrame(columns=['RT', 'mz', 'inty']) + + +def plot_bpc_tic() -> go.Figure: + """Plot the base peak and total ion chromatogram (TIC). + + Returns: + A plotly Figure object containing the BPC and TIC plot. + """ + fig = go.Figure() + max_int = 0 + if st.session_state.view_tic: + df = st.session_state.view_ms1.groupby("RT").sum().reset_index() + df["type"] = "TIC" + if df["inty"].max() > max_int: + max_int = df["inty"].max() + fig = df.plot( + backend="ms_plotly", + kind="chromatogram", + x="RT", + y="inty", + by="type", + color="#f24c5c", + show_plot=False, + grid=False, + aggregate_duplicates=True, + ) + if st.session_state.view_bpc: + df = st.session_state.view_ms1.groupby("RT").max().reset_index() + df["type"] = "BPC" + if df["inty"].max() > max_int: + max_int = df["inty"].max() + fig = df.plot( + backend="ms_plotly", + kind="chromatogram", + x="RT", + y="inty", + by="type", + color="#2d3a9d", + show_plot=False, + grid=False, + aggregate_duplicates=True, + ) + if st.session_state.view_eic: + df = st.session_state.view_ms1 + target_value = st.session_state.view_eic_mz.strip().replace(",", ".") + try: + target_value = float(target_value) + ppm_tolerance = st.session_state.view_eic_ppm + tolerance = (target_value * ppm_tolerance) / 1e6 + + # Filter the DataFrame + df_eic = df[ + (df["mz"] >= target_value - tolerance) + & (df["mz"] <= target_value + tolerance) + ].copy() + if not df_eic.empty: + df_eic.loc[:, "type"] = "XIC" + if df_eic["inty"].max() > max_int: + max_int = df_eic["inty"].max() + fig = df_eic.plot( + backend="ms_plotly", + kind="chromatogram", + x="RT", + y="inty", + by="type", + color="#f6bf26", + show_plot=False, + grid=False, + aggregate_duplicates=True, + ) + except ValueError: + st.error("Invalid m/z value for XIC provided. Please enter a valid number.") + + fig.update_yaxes(range=[0, max_int]) + fig.update_layout( + title=f"{st.session_state.view_selected_file}", + xaxis_title="retention time (s)", + yaxis_title="intensity", + plot_bgcolor="rgb(255,255,255)", + height=500, + ) + fig.layout.template = "plotly_white" + return fig + + +@st.cache_resource +def plot_ms_spectrum(df, title, bin_peaks, num_x_bins): + fig = df.plot( + kind="spectrum", + backend="ms_plotly", + x="mz", + y="intensity", + color="#2d3a9d", + title=title, + show_plot=False, + grid=False, + bin_peaks=bin_peaks, + num_x_bins=num_x_bins, + aggregate_duplicates=True, + ) + fig.update_layout( + template="plotly_white", dragmode="select", plot_bgcolor="rgb(255,255,255)" + ) + return fig + + +@st.fragment +def view_peak_map(): + df = st.session_state.view_ms1 + if "view_peak_map_selection" in st.session_state: + box = st.session_state.view_peak_map_selection.selection.box + if box: + df = st.session_state.view_ms1.copy() + df = df[df["RT"] > box[0]["x"][0]] + df = df[df["mz"] > box[0]["y"][1]] + df = df[df["mz"] < box[0]["y"][0]] + df = df[df["RT"] < box[0]["x"][1]] + if len(df) == 0: + return + peak_map = df.plot( + kind="peakmap", + x="RT", + y="mz", + z="inty", + title=st.session_state.view_selected_file, + grid=False, + show_plot=False, + bin_peaks=True, + backend="ms_plotly", + aggregate_duplicates=True, + ) + peak_map.update_layout(template="simple_white", dragmode="select") + c1, c2 = st.columns(2) + with c1: + st.info( + "πŸ’‘ Zoom in via rectangular selection for more details and 3D plot. Double click plot to zoom back out." + ) + show_fig( + peak_map, + f"peak_map_{st.session_state.view_selected_file}", + selection_session_state_key="view_peak_map_selection", + ) + with c2: + if df.shape[0] < 2500: + peak_map_3D = df.plot( + kind="peakmap", + plot_3d=True, + backend="ms_plotly", + x="RT", + y="mz", + z="inty", + zlabel="Intensity", + title="", + show_plot=False, + grid=False, + bin_peaks=st.session_state.spectrum_bin_peaks, + num_x_bins=st.session_state.spectrum_num_bins, + height=650, + width=900, + aggregate_duplicates=True, + ) + st.plotly_chart(peak_map_3D, use_container_width=True) + + +@st.fragment +def view_spectrum(): + cols = st.columns([0.34, 0.66]) + with cols[0]: + df = st.session_state.view_spectra.copy() + df["spectrum ID"] = df.index + 1 + index = display_large_dataframe( + df, + column_order=[ + "spectrum ID", + "RT", + "MS level", + "max intensity m/z", + "precursor m/z", + ], + selection_mode="single-row", + on_select="rerun", + use_container_width=True, + hide_index=True, + ) + with cols[1]: + if (index is not None) and (len(df) != 0): + df = st.session_state.view_spectra.iloc[index] + if "view_spectrum_selection" in st.session_state: + box = st.session_state.view_spectrum_selection.selection.box + if box: + mz_min, mz_max = sorted(box[0]["x"]) + mask = (df["mzarray"] > mz_min) & (df["mzarray"] < mz_max) + df["intarray"] = df["intarray"][mask] + df["mzarray"] = df["mzarray"][mask] + + if df["mzarray"].size > 0: + title = f"{st.session_state.view_selected_file} spec={index+1} mslevel={df['MS level']}" + if df["precursor m/z"] > 0: + title += f" precursor m/z: {round(df['precursor m/z'], 4)}" + + df_selected = pd.DataFrame( + { + "mz": df["mzarray"], + "intensity": df["intarray"], + } + ) + df_selected["RT"] = df["RT"] + df_selected["MS level"] = df["MS level"] + df_selected["precursor m/z"] = df["precursor m/z"] + df_selected["max intensity m/z"] = df["max intensity m/z"] + + fig = plot_ms_spectrum( + df_selected, + title, + st.session_state.spectrum_bin_peaks, + st.session_state.spectrum_num_bins, + ) + + show_fig(fig, title.replace(" ", "_"), True, "view_spectrum_selection") + else: + st.session_state.pop("view_spectrum_selection") + st.rerun() + else: + st.info("πŸ’‘ Select rows in the spectrum table to display plot.") + + +@st.fragment() +def view_bpc_tic(): + cols = st.columns(5) + cols[0].checkbox( + "Total Ion Chromatogram (TIC)", True, key="view_tic", help="Plot TIC." + ) + cols[1].checkbox( + "Base Peak Chromatogram (BPC)", True, key="view_bpc", help="Plot BPC." + ) + cols[2].checkbox( + "Extracted Ion Chromatogram (EIC/XIC)", + True, + key="view_eic", + help="Plot extracted ion chromatogram with specified m/z.", + ) + cols[3].text_input( + "XIC m/z", + "235.1189", + help="m/z for XIC calculation.", + key="view_eic_mz", + ) + cols[4].number_input( + "XIC ppm tolerance", + 0.1, + 50.0, + 10.0, + 1.0, + help="Tolerance for XIC calculation (ppm).", + key="view_eic_ppm", + ) + fig = plot_bpc_tic() + show_fig(fig, f"BPC-TIC-{st.session_state.view_selected_file}") diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 042c5e1..6479cb0 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -272,9 +272,10 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool # Load flag parameter names: params.json takes priority (survives session restart), # session_state is the live fallback during the current session. flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {}) - if not flag_map: - flag_map = st.session_state.get("_topp_flag_params", {}) - flag_params: set = set(flag_map.get(params_key, [])) + flag_list = flag_map.get(params_key) + if flag_list is None: + flag_list = st.session_state.get("_topp_flag_params", {}).get(params_key, []) + flag_params: set = set(flag_list) # Construct commands for each process for i in range(n_processes): @@ -305,12 +306,14 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool if is_enabled: command += [f"-{k}"] continue - # Regular parameter: skip empty/None, append value otherwise - if v == "" or v is None: + # Regular parameter: skip empty/None/empty-list, append value otherwise + if v == "" or v is None or (isinstance(v, list) and not v): continue command += [f"-{k}"] if isinstance(v, str) and "\n" in v: command += v.split("\n") + elif isinstance(v, list): + command += [str(x) for x in v] else: command += [str(v)] # Add custom parameters @@ -323,7 +326,7 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool if is_enabled: command += [f"-{k}"] continue - if v == "" or v is None: + if v == "" or v is None or (isinstance(v, list) and not v): continue command += [f"-{k}"] if isinstance(v, list): diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 77ca01d..d426e3a 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -834,36 +834,44 @@ def input_TOPP( reactive: bool = False, ) -> None: """ - Wrapper for TOPP parameter input. When `reactive` is True the - implementation is rendered directly in the parent context so changes - trigger a parent re-render; otherwise the widgets are rendered inside - a `st.fragment` to isolate reruns for performance. + Generates input widgets for TOPP tool parameters dynamically based on the tool's + .ini file. Supports excluding specific parameters and adjusting the layout. + File input and output parameters are excluded. + + Args: + topp_tool_name (str): The name of the TOPP tool for which to generate inputs. + num_cols (int, optional): Number of columns to use for the layout. Defaults to 3. + exclude_parameters (List[str], optional): List of parameter names to exclude from the widget. Defaults to an empty list. + include_parameters (List[str], optional): List of parameter names to include in the widget. Defaults to an empty list. + flag_parameters (List[str], optional): List of parameter names that should + be treated as no-value CLI flags during command construction. + display_tool_name (bool, optional): Whether to display the TOPP tool name. Defaults to True. + display_subsections (bool, optional): Whether to split parameters into subsections based on the prefix. Defaults to True. + display_subsection_tabs (bool, optional): Whether to display main subsections in separate tabs (if more than one main section). Defaults to False. + custom_defaults (dict, optional): Dictionary of custom defaults to use. Defaults to an empty dict. + tool_instance_name (str, optional): A unique instance name for this tool + invocation. Allows multiple instances of the same TOPP tool with + independent parameters (e.g., two IDFilter calls). If not provided, + defaults to topp_tool_name. The instance name is used for session + state keys and parameter storage, while topp_tool_name is used for + the actual tool executable and ini file creation. + reactive (bool, optional): If True, widget changes trigger the parent + section to re-render, enabling conditional UI based on this widget's + value. Use when downstream UI depends on a parameter value (e.g., + TMT type driving channel count). Default is False. """ if reactive: - return self._input_TOPP_impl( - topp_tool_name=topp_tool_name, - num_cols=num_cols, - exclude_parameters=exclude_parameters, - include_parameters=include_parameters, - flag_parameters=flag_parameters, - display_tool_name=display_tool_name, - display_subsections=display_subsections, - display_subsection_tabs=display_subsection_tabs, - custom_defaults=custom_defaults, - tool_instance_name=tool_instance_name, + self._input_TOPP_impl( + topp_tool_name, num_cols, exclude_parameters, include_parameters, + flag_parameters, display_tool_name, display_subsections, + display_subsection_tabs, custom_defaults, tool_instance_name, + ) + else: + self._input_TOPP_fragmented( + topp_tool_name, num_cols, exclude_parameters, include_parameters, + flag_parameters, display_tool_name, display_subsections, + display_subsection_tabs, custom_defaults, tool_instance_name, ) - return self._input_TOPP_fragmented( - topp_tool_name=topp_tool_name, - num_cols=num_cols, - exclude_parameters=exclude_parameters, - include_parameters=include_parameters, - flag_parameters=flag_parameters, - display_tool_name=display_tool_name, - display_subsections=display_subsections, - display_subsection_tabs=display_subsection_tabs, - custom_defaults=custom_defaults, - tool_instance_name=tool_instance_name, - ) @st.fragment def _input_TOPP_fragmented( @@ -879,17 +887,10 @@ def _input_TOPP_fragmented( custom_defaults: dict = {}, tool_instance_name: str = None, ) -> None: - return self._input_TOPP_impl( - topp_tool_name=topp_tool_name, - num_cols=num_cols, - exclude_parameters=exclude_parameters, - include_parameters=include_parameters, - flag_parameters=flag_parameters, - display_tool_name=display_tool_name, - display_subsections=display_subsections, - display_subsection_tabs=display_subsection_tabs, - custom_defaults=custom_defaults, - tool_instance_name=tool_instance_name, + self._input_TOPP_impl( + topp_tool_name, num_cols, exclude_parameters, include_parameters, + flag_parameters, display_tool_name, display_subsections, + display_subsection_tabs, custom_defaults, tool_instance_name, ) def _input_TOPP_impl( @@ -905,27 +906,7 @@ def _input_TOPP_impl( custom_defaults: dict = {}, tool_instance_name: str = None, ) -> None: - """ - Generates input widgets for TOPP tool parameters dynamically based on the tool's - .ini file. Supports excluding specific parameters and adjusting the layout. - File input and output parameters are excluded. - - Args: - topp_tool_name (str): The name of the TOPP tool for which to generate inputs. - num_cols (int, optional): Number of columns to use for the layout. Defaults to 3. - exclude_parameters (List[str], optional): List of parameter names to exclude from the widget. Defaults to an empty list. - include_parameters (List[str], optional): List of parameter names to include in the widget. Defaults to an empty list. - display_tool_name (bool, optional): Whether to display the TOPP tool name. Defaults to True. - display_subsections (bool, optional): Whether to split parameters into subsections based on the prefix. Defaults to True. - display_subsection_tabs (bool, optional): Whether to display main subsections in separate tabs (if more than one main section). Defaults to False. - custom_defaults (dict, optional): Dictionary of custom defaults to use. Defaults to an empty dict. - tool_instance_name (str, optional): A unique instance name for this tool - invocation. Allows multiple instances of the same TOPP tool with - independent parameters (e.g., two IDFilter calls). If not provided, - defaults to topp_tool_name. The instance name is used for session - state keys and parameter storage, while topp_tool_name is used for - the actual tool executable and ini file creation. - """ + """Internal implementation of input_TOPP - contains all the widget logic.""" # Default instance name to the tool name when not provided if tool_instance_name is None: tool_instance_name = topp_tool_name diff --git a/src/workflow/WorkflowManager.py b/src/workflow/WorkflowManager.py index 302b079..856ae56 100644 --- a/src/workflow/WorkflowManager.py +++ b/src/workflow/WorkflowManager.py @@ -206,10 +206,9 @@ def stop_workflow(self) -> bool: return self._stop_local_workflow() def _stop_local_workflow(self) -> bool: - """Stop locally running workflow process - Windows Compatible""" + """Stop locally running workflow process""" import os import signal - import platform pid_dir = self.executor.pid_dir if not pid_dir.exists(): @@ -219,18 +218,11 @@ def _stop_local_workflow(self) -> bool: for pid_file in pid_dir.iterdir(): try: pid = int(pid_file.name) - # Windows - if platform.system() == "Windows": - os.system(f"taskkill /F /T /PID {pid}") - else: - # Linux/macOS - os.kill(pid, signal.SIGTERM) - + os.kill(pid, signal.SIGTERM) pid_file.unlink() stopped = True - except (ValueError, ProcessLookupError, PermissionError, OSError): - if pid_file.exists(): - pid_file.unlink() + except (ValueError, ProcessLookupError, PermissionError): + pid_file.unlink() # Clean up stale PID file # Clean up the pid directory shutil.rmtree(pid_dir, ignore_errors=True) diff --git a/test.py b/test.py new file mode 100644 index 0000000..8a2a3ad --- /dev/null +++ b/test.py @@ -0,0 +1,24 @@ +# test_my_math.py +import unittest +from urllib.request import urlretrieve + +from src.simpleworkflow import generate_random_table +from src.mzmlfileworkflow import mzML_file_get_num_spectra + +from pathlib import Path + +class TestSimpleWorkflow(unittest.TestCase): + def test_workflow(self): + result = generate_random_table(2, 3).shape + self.assertEqual(result, (2,3), "Expected dataframe shape.") + +class TestComplexWorkflow(unittest.TestCase): + def test_workflow(self): + # load data from url + urlretrieve("https://raw.githubusercontent.com/OpenMS/streamlit-template/main/example-data/mzML/Treatment.mzML", "testfile.mzML") + result = mzML_file_get_num_spectra("testfile.mzML") + Path("testfile.mzML").unlink() + self.assertEqual(result, 786, "Expected dataframe shape.") + +if __name__ == '__main__': + unittest.main() diff --git a/test_gui.py b/test_gui.py index 0ab2711..0485bae 100644 --- a/test_gui.py +++ b/test_gui.py @@ -1,40 +1,141 @@ -import json - -import pytest from streamlit.testing.v1 import AppTest +import pytest +from src import fileupload +import json +from pathlib import Path +import shutil -# Pages that AppTest.from_file can load in isolation. Pages using st.page_link -# require streamlit's navigation context (only set up when app.py runs), so they -# are covered indirectly by test_app_loads below. -DIRECTLY_TESTABLE_PAGES = [ - "content/workflow_fileupload.py", - "content/workflow_configure.py", - "content/workflow_run.py", - "content/results_library.py", - "content/results_proteomicslfq.py", -] +@pytest.fixture +def launch(request): + test = AppTest.from_file(request.param) -def _init(apptest): + ## Initialize session state ## with open("settings.json", "r") as f: - apptest.session_state.settings = json.load(f) - apptest.session_state.settings["test"] = True - apptest.secrets["workspace"] = "test" - return apptest + test.session_state.settings = json.load(f) + test.session_state.settings["test"] = True + test.secrets["workspace"] = "test" + return test -@pytest.fixture -def launch(request): - return _init(AppTest.from_file(request.param)) +# Test launching of all pages +@pytest.mark.parametrize( + "launch", + ( + # "content/quickstart.py", # NOTE: this page does not work due to streamlit.errors.StreamlitPageNotFoundError error + "content/documentation.py", + "content/topp_workflow_file_upload.py", + "content/topp_workflow_parameter.py", + "content/topp_workflow_execution.py", + "content/topp_workflow_results.py", + "content/file_upload.py", + "content/raw_data_viewer.py", + "content/run_example_workflow.py", + "content/download_section.py", + "content/simple_workflow.py", + "content/run_subprocess.py", + ), + indirect=True, +) +def test_launch(launch): + """Test if all pages can be launched without errors.""" + launch.run(timeout=30) # Increased timeout from 10 to 30 seconds + assert not launch.exception + + +########### PAGE SPECIFIC TESTS ############ +@pytest.mark.parametrize( + "launch,selection", + [ + ("content/documentation.py", "User Guide"), + ("content/documentation.py", "Installation"), + ( + "content/documentation.py", + "Developers Guide: How to build app based on this template", + ), + ("content/documentation.py", "Developers Guide: TOPP Workflow Framework"), + ("content/documentation.py", "Developer Guide: Windows Executables"), + ("content/documentation.py", "Developers Guide: Deployment"), + ("content/documentation.py", "Developers Guide: Kubernetes Deployment"), + ], + indirect=["launch"], +) +def test_documentation(launch, selection): + launch.run() + launch.selectbox[0].select(selection).run() + assert not launch.exception + + +@pytest.mark.parametrize("launch", ["content/file_upload.py"], indirect=True) +def test_file_upload_load_example(launch): + launch.run() + for i in launch.tabs: + if i.label == "Example Data": + i.button[0].click().run() + assert not launch.exception + + +# NOTE: All tabs are automatically checked +@pytest.mark.parametrize( + "launch,example", + [ + ("content/raw_data_viewer.py", "Blank.mzML"), + ("content/raw_data_viewer.py", "Treatment.mzML"), + ("content/raw_data_viewer.py", "Pool.mzML"), + ("content/raw_data_viewer.py", "Control.mzML"), + ], + indirect=["launch"], +) +def test_view_raw_ms_data(launch, example): + launch.run(timeout=30) # Increased timeout from 10 to 30 seconds + + ## Load Example file, based on implementation of fileupload.load_example_mzML_files() ### + mzML_dir = Path(launch.session_state.workspace, "mzML-files") + # Copy files from example-data/mzML to workspace mzML directory, add to selected files + for f in Path("example-data", "mzML").glob("*.mzML"): + try: + shutil.copy(f, mzML_dir) + except shutil.SameFileError: + pass # File already exists as a symlink to the same source (on Linux) + launch.run() -@pytest.mark.parametrize("launch", DIRECTLY_TESTABLE_PAGES, indirect=True) -def test_page_loads(launch): - launch.run(timeout=30) + ## TODO: Figure out a way to select a spectrum to be displayed + launch.selectbox[0].select(example).run() assert not launch.exception -def test_app_loads(): - app = _init(AppTest.from_file("app.py")) - app.run(timeout=30) - assert not app.exception +@pytest.mark.parametrize( + "launch,example", + [ + ("content/run_example_workflow.py", ["Blank"]), + ("content/run_example_workflow.py", ["Treatment"]), + ("content/run_example_workflow.py", ["Pool"]), + ("content/run_example_workflow.py", ["Control"]), + ("content/run_example_workflow.py", ["Control", "Blank"]), + ], + indirect=["launch"], +) +def test_run_workflow(launch, example): + launch.run() + ## Load Example file, based on implementation of fileupload.load_example_mzML_files() ### + mzML_dir = Path(launch.session_state.workspace, "mzML-files") + + # Copy files from example-data/mzML to workspace mzML directory, add to selected files + for f in Path("example-data", "mzML").glob("*.mzML"): + try: + shutil.copy(f, mzML_dir) + except shutil.SameFileError: + pass # File already exists as a symlink to the same source (on Linux) + launch.run() + + ## Select experiments to process + for e in example: + launch.multiselect[0].select(e) + + launch.run() + assert not launch.exception + + # Press the "Run Workflow" button + launch.button[1].click().run(timeout=60) + assert not launch.exception diff --git a/tests/test_legal_links.py b/tests/test_legal_links.py new file mode 100644 index 0000000..a38e201 --- /dev/null +++ b/tests/test_legal_links.py @@ -0,0 +1,160 @@ +""" +Tests for get_legal_links() in src/common/common.py. + +get_legal_links() resolves the Impressum / Privacy Policy / Terms of Use URLs +shown in the sidebar footer (on every page) and the privacy-policy link wired +into the GDPR consent banner. It merges the optional "legal_links" object from +settings.json over the built-in official-OpenMS defaults so that: + + * apps built from a settings.json without a "legal_links" key still inherit + working legal links by default, + * a self-hosting fork can override any or all of the three URLs, + * an empty/blank override value never erases a default. + +Streamlit (and the other heavy runtime deps pulled in by common.py) are mocked +before import so the helper can be unit-tested without a running Streamlit app, +mirroring tests/test_parameter_presets.py. +""" +import os +import sys +from unittest.mock import MagicMock + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + + +class FakeSessionState(dict): + """Minimal stand-in for Streamlit's SessionState. + + Supports both attribute access (``state.settings``) and item/membership + access (``"settings" in state``), exactly like the real SessionState that + common.py relies on. + """ + + def __getattr__(self, name): + try: + return self[name] + except KeyError as exc: + raise AttributeError(name) from exc + + def __setattr__(self, name, value): + self[name] = value + + +# Mock streamlit (with a SessionState-like session_state) and the other heavy +# imports pulled in by src/common/common.py, so importing get_legal_links here +# doesn't require a running Streamlit app context. +# +# IMPORTANT: these mocks are installed into sys.modules only for the duration of +# the import below and then restored, so they don't leak into other test modules +# (e.g. the AppTest-based tests that need the real `streamlit` package). This +# mirrors the pattern in tests/test_parameter_presets.py. +mock_streamlit = MagicMock() +mock_streamlit.session_state = FakeSessionState() + +_MOCKED_MODULES = { + "streamlit": mock_streamlit, + "streamlit.components": MagicMock(), + "streamlit.components.v1": MagicMock(), + "streamlit.source_util": MagicMock(), + "pandas": MagicMock(), + "psutil": MagicMock(), + # Local submodules with their own heavy deps (e.g. the captcha image library). + "src.common.captcha_": MagicMock(), + "src.common.admin": MagicMock(), +} +_saved_modules = {name: sys.modules.get(name) for name in _MOCKED_MODULES} +sys.modules.update(_MOCKED_MODULES) + +# Force a FRESH import of src.common.common under the streamlit mock, even if an +# earlier test module (e.g. test_gui.py) already imported the real-streamlit-bound +# version. Save whatever was cached first so we can restore it afterwards. +_saved_common = sys.modules.pop("src.common.common", None) + +from src.common.common import get_legal_links, DEFAULT_LEGAL_LINKS # noqa: E402 + +# Restore the real modules (or remove ones that weren't present) so that other +# test modules get the genuine packages. +for _name, _orig in _saved_modules.items(): + if _orig is None: + sys.modules.pop(_name, None) + else: + sys.modules[_name] = _orig +# Restore the original cached common module (the real-streamlit-bound one, if +# any) so AppTest-based test modules keep getting the genuine package. +# get_legal_links keeps working: it holds a reference to the freshly-imported +# mock-bound module's globals (and the same `mock_streamlit` object the tests +# mutate). +if _saved_common is None: + sys.modules.pop("src.common.common", None) +else: + sys.modules["src.common.common"] = _saved_common + + +def setup_function(_): + """Reset session_state before each test for isolation.""" + mock_streamlit.session_state = FakeSessionState() + + +def test_defaults_point_to_openms(): + """The built-in defaults are the official OpenMS pages.""" + assert DEFAULT_LEGAL_LINKS == { + "impressum": "https://openms.de/impressum", + "privacy": "https://openms.de/privacy", + "terms": "https://openms.de/terms", + } + + +def test_defaults_when_settings_not_loaded(): + """No settings loaded at all -> defaults, no crash.""" + mock_streamlit.session_state = FakeSessionState() + assert get_legal_links() == DEFAULT_LEGAL_LINKS + + +def test_defaults_when_no_legal_links_key(): + """settings present but without 'legal_links' -> all OpenMS defaults.""" + mock_streamlit.session_state = FakeSessionState({"settings": {}}) + assert get_legal_links() == DEFAULT_LEGAL_LINKS + + +def test_overrides_replace_defaults(): + """A fork's custom legal_links replace every default.""" + mock_streamlit.session_state = FakeSessionState( + { + "settings": { + "legal_links": { + "impressum": "https://acme.example/impressum", + "privacy": "https://acme.example/privacy", + "terms": "https://acme.example/terms", + } + } + } + ) + assert get_legal_links() == { + "impressum": "https://acme.example/impressum", + "privacy": "https://acme.example/privacy", + "terms": "https://acme.example/terms", + } + + +def test_partial_override_keeps_other_defaults(): + """Overriding only one link leaves the others at their OpenMS default.""" + mock_streamlit.session_state = FakeSessionState( + {"settings": {"legal_links": {"impressum": "https://acme.example/impressum"}}} + ) + links = get_legal_links() + assert links["impressum"] == "https://acme.example/impressum" + assert links["privacy"] == DEFAULT_LEGAL_LINKS["privacy"] + assert links["terms"] == DEFAULT_LEGAL_LINKS["terms"] + + +def test_empty_or_none_override_falls_back_to_default(): + """A blank/None override must not erase the default for that key.""" + mock_streamlit.session_state = FakeSessionState( + {"settings": {"legal_links": {"privacy": "", "impressum": None}}} + ) + links = get_legal_links() + assert links["privacy"] == DEFAULT_LEGAL_LINKS["privacy"] + assert links["impressum"] == DEFAULT_LEGAL_LINKS["impressum"] + assert links["terms"] == DEFAULT_LEGAL_LINKS["terms"] diff --git a/tests/test_parameter_defaults.py b/tests/test_parameter_defaults.py new file mode 100644 index 0000000..9ccd679 --- /dev/null +++ b/tests/test_parameter_defaults.py @@ -0,0 +1,363 @@ +""" +Tests for get_merged_params() and the refactored get_topp_parameters(). + +This module verifies the three-layer parameter merge: + ini defaults < _defaults < user overrides +""" +import os +import sys +import json +import pytest +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# Mock streamlit before importing ParameterManager so that the imported module +# uses a controllable `st.session_state` (a plain dict) instead of the real one, +# which requires a running Streamlit app context. +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +# Temporarily replace streamlit in sys.modules so that ParameterManager's +# `import streamlit as st` picks up the mock. Restore immediately after import +# so other test files (e.g., test_gui.py AppTest) get the real streamlit. +_original_streamlit = sys.modules.get('streamlit') +sys.modules['streamlit'] = mock_streamlit + +from src.workflow.ParameterManager import ParameterManager + +if _original_streamlit is not None: + sys.modules['streamlit'] = _original_streamlit +else: + sys.modules.pop('streamlit', None) + +# Remove cached src.workflow modules that were imported with mocked streamlit so +# that AppTest (in test_gui.py) re-imports them fresh with the real package. +for _key in list(sys.modules.keys()): + if _key.startswith('src.workflow'): + sys.modules.pop(_key, None) + + +@pytest.fixture(autouse=True) +def reset_streamlit_state(): + """Reset mock streamlit session state before each test.""" + mock_streamlit.session_state.clear() + yield + + +@pytest.fixture +def temp_workflow_dir(): + """Create a temporary workflow directory for testing.""" + with tempfile.TemporaryDirectory() as tmpdir: + workflow_dir = Path(tmpdir) / "test-workflow" + workflow_dir.mkdir() + ini_dir = workflow_dir / "ini" + ini_dir.mkdir() + yield workflow_dir + + +class TestGetMergedParams: + """Tests for ParameterManager.get_merged_params().""" + + def test_returns_ini_params_when_no_json(self, temp_workflow_dir): + """ini params returned when params.json doesn't exist.""" + pm = ParameterManager(temp_workflow_dir) + ini_params = {"algorithm:param": 1.0, "algorithm:other": "hello"} + + result = pm.get_merged_params("SomeTool", ini_params=ini_params) + + assert result == {"algorithm:param": 1.0, "algorithm:other": "hello"} + + def test_defaults_override_ini(self, temp_workflow_dir): + """_defaults layer overrides ini values.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool", ini_params={"algorithm:param": 1.0}) + + assert result["algorithm:param"] == 42.0 + + def test_user_overrides_defaults(self, temp_workflow_dir): + """User overrides take priority over _defaults.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}}, + "SomeTool": {"algorithm:param": 99.0} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool", ini_params={"algorithm:param": 1.0}) + + assert result["algorithm:param"] == 99.0 + + def test_full_three_layer_merge(self, temp_workflow_dir): + """All three layers merge correctly: ini < _defaults < user.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": { + "SomeTool": { + "algorithm:param_a": 10.0, # overrides ini + "algorithm:param_b": 20.0, # overrides ini, NOT overridden by user + } + }, + "SomeTool": { + "algorithm:param_a": 99.0, # overrides _defaults + "algorithm:param_c": 55.0, # only in user + } + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + ini_params = { + "algorithm:param_a": 1.0, + "algorithm:param_b": 2.0, + "algorithm:param_d": 3.0, # only in ini + } + + result = pm.get_merged_params("SomeTool", ini_params=ini_params) + + assert result["algorithm:param_a"] == 99.0 # user wins + assert result["algorithm:param_b"] == 20.0 # _defaults wins over ini + assert result["algorithm:param_c"] == 55.0 # user-only key present + assert result["algorithm:param_d"] == 3.0 # ini-only key present + + def test_no_ini_params(self, temp_workflow_dir): + """Works when ini_params is None.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}}, + "SomeTool": {"algorithm:other": 7.0} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool") + + assert result["algorithm:param"] == 42.0 + assert result["algorithm:other"] == 7.0 + + def test_different_instances_same_tool(self, temp_workflow_dir): + """Different instance names get independent _defaults and user overrides.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": { + "IDFilter_step1": {"score:min": 0.05}, + "IDFilter_step2": {"score:min": 0.01}, + }, + "IDFilter_step1": {"score:min": 0.001}, + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result1 = pm.get_merged_params("IDFilter_step1", ini_params={"score:min": 0.5}) + result2 = pm.get_merged_params("IDFilter_step2", ini_params={"score:min": 0.5}) + + assert result1["score:min"] == 0.001 # user override for step1 + assert result2["score:min"] == 0.01 # _defaults for step2 (no user override) + + def test_empty_params_json(self, temp_workflow_dir): + """Returns empty dict when params.json is empty and no ini_params.""" + pm = ParameterManager(temp_workflow_dir) + with open(pm.params_file, "w") as f: + json.dump({}, f) + + result = pm.get_merged_params("SomeTool") + + assert result == {} + + +class TestGetToppParametersWithDefaults: + + def test_get_topp_parameters_includes_defaults(self, temp_workflow_dir): + """get_topp_parameters merges _defaults between ini and user values.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}}, + "SomeTool": {"algorithm:other": 99.0} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool", ini_params={"algorithm:param": 1.0}) + assert result["algorithm:param"] == 42.0 + assert result["algorithm:other"] == 99.0 + + +class TestDefaultsSeeding: + + def test_seed_writes_defaults_to_params_json(self, temp_workflow_dir): + """Seeding creates _defaults entry in params.json.""" + pm = ParameterManager(temp_workflow_dir) + custom_defaults = {"param_a": 10.0, "param_b": "fast"} + + # Simulate what input_TOPP seeding does + params = pm.get_parameters_from_json() + if "_defaults" not in params: + params["_defaults"] = {} + params["_defaults"]["MyTool"] = custom_defaults + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Verify + loaded = pm.get_parameters_from_json() + assert loaded["_defaults"]["MyTool"] == {"param_a": 10.0, "param_b": "fast"} + + def test_seed_is_idempotent(self, temp_workflow_dir): + """Seeding the same tool twice overwrites cleanly.""" + pm = ParameterManager(temp_workflow_dir) + + # First seed + params = {"_defaults": {"Tool": {"p1": 1.0}}, "other_key": "keep"} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Second seed with updated defaults + params = pm.get_parameters_from_json() + params["_defaults"]["Tool"] = {"p1": 2.0} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + loaded = pm.get_parameters_from_json() + assert loaded["_defaults"]["Tool"]["p1"] == 2.0 + assert loaded["other_key"] == "keep" + + def test_seed_multiple_instances(self, temp_workflow_dir): + """Different instances of the same tool get independent _defaults.""" + pm = ParameterManager(temp_workflow_dir) + params = { + "_defaults": { + "IDFilter_strict": {"score:pep": 0.01}, + "IDFilter_lenient": {"score:pep": 0.05}, + } + } + with open(pm.params_file, "w") as f: + json.dump(params, f) + + loaded = pm.get_parameters_from_json() + assert loaded["_defaults"]["IDFilter_strict"]["score:pep"] == 0.01 + assert loaded["_defaults"]["IDFilter_lenient"]["score:pep"] == 0.05 + + +try: + import pyopenms as poms + HAS_PYOPENMS = True +except ImportError: + HAS_PYOPENMS = False + + +@pytest.mark.skipif(not HAS_PYOPENMS, reason="pyopenms not available") +class TestSaveParametersWithDefaults: + + def _create_fake_ini(self, pm, tool_name, params_dict): + """Create a fake .ini file with given parameters.""" + param = poms.Param() + for key, value in params_dict.items(): + param.setValue(f"{tool_name}:1:{key}".encode(), value) + poms.ParamXMLFile().store(str(Path(pm.ini_dir, f"{tool_name}.ini")), param) + + def test_value_matching_custom_default_not_saved(self, temp_workflow_dir): + """A value equal to the _defaults entry should not be saved as a user override.""" + pm = ParameterManager(temp_workflow_dir) + + # Create a fake ini with a default value + self._create_fake_ini(pm, "Tool", {"param_a": 10.0}) + + # Pre-seed _defaults with a different value than ini + params = {"_defaults": {"Tool": {"param_a": 42.0}}} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Session state has value matching the custom default (42.0), not the ini default (10.0) + mock_streamlit.session_state[f"{pm.topp_param_prefix}Tool:1:param_a"] = 42.0 + mock_streamlit.session_state["_topp_tool_instance_map"] = {"Tool": "Tool"} + + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # param_a should NOT appear under Tool (it matches the _defaults value) + assert "param_a" not in saved.get("Tool", {}) + # _defaults should still be present + assert saved["_defaults"]["Tool"]["param_a"] == 42.0 + + def test_value_different_from_custom_default_saved(self, temp_workflow_dir): + """A value different from _defaults entry should be saved as user override.""" + pm = ParameterManager(temp_workflow_dir) + + # Create a fake ini with a default value + self._create_fake_ini(pm, "Tool", {"param_a": 10.0}) + + params = {"_defaults": {"Tool": {"param_a": 42.0}}} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + mock_streamlit.session_state[f"{pm.topp_param_prefix}Tool:1:param_a"] = 99.0 + mock_streamlit.session_state["_topp_tool_instance_map"] = {"Tool": "Tool"} + + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + assert saved["Tool"]["param_a"] == 99.0 + + +class TestNonDefaultParamsSummaryDefaults: + + def test_defaults_key_excluded_from_classification(self): + """_defaults dict should not appear as a TOPP tool in the summary.""" + params = { + "_defaults": {"Tool": {"p1": 10}}, + "Tool": {"p1": 20}, + "general_param": "value" + } + # Simulate the classification logic + topp = {} + general = {} + for k, v in params.items(): + if k == "_defaults": + continue + if isinstance(v, dict): + topp[k] = v + else: + general[k] = v + + assert "_defaults" not in topp + assert "Tool" in topp + assert "general_param" in general + + def test_defaults_merged_into_summary(self): + """_defaults values should appear in summary merged with user overrides.""" + params = { + "_defaults": { + "ToolA": {"p1": 10, "p2": 20}, + "ToolB": {"p3": 30} + }, + "ToolA": {"p1": 99} + } + # Simulate the merge logic for summary + topp = {} + for k, v in params.items(): + if k == "_defaults": + continue + if isinstance(v, dict): + topp[k] = v + + defaults = params.get("_defaults", {}) + for tool_name, default_vals in defaults.items(): + if tool_name not in topp: + topp[tool_name] = {} + topp[tool_name] = {**default_vals, **topp.get(tool_name, {})} + + assert topp["ToolA"] == {"p1": 99, "p2": 20} # user override wins for p1 + assert topp["ToolB"] == {"p3": 30} # defaults-only tool appears diff --git a/tests/test_queue_manager_cancel.py b/tests/test_queue_manager_cancel.py index 0f87708..c8ef44a 100644 --- a/tests/test_queue_manager_cancel.py +++ b/tests/test_queue_manager_cancel.py @@ -153,9 +153,7 @@ def test_stopped_status_is_mapped_in_get_job_info(monkeypatch): info = qm.get_job_info("stopped-job") assert info is not None - assert info.status == __import__( - "src.workflow.QueueManager", fromlist=["JobStatus"] - ).JobStatus.CANCELED, ( + assert info.status.name == "CANCELED", ( "RQ 'stopped' status should be reported as CANCELED to the UI; " "otherwise stopped jobs appear stuck in 'queued'." ) diff --git a/tests/test_run_subprocess.py b/tests/test_run_subprocess.py new file mode 100644 index 0000000..cd6889a --- /dev/null +++ b/tests/test_run_subprocess.py @@ -0,0 +1,37 @@ +import pytest +import time +from streamlit.testing.v1 import AppTest + +@pytest.fixture +def launch(): + """Launch the Run Subprocess Streamlit page for testing.""" + + app = AppTest.from_file("content/run_subprocess.py") + app.run(timeout=10) + return app + +def test_file_selection(launch): + """Ensure a file can be selected from the dropdown.""" + launch.run() + + assert len(launch.selectbox) > 0, "No file selection dropdown found!" + + if len(launch.selectbox[0].options) > 0: + launch.selectbox[0].select(launch.selectbox[0].options[0]) + launch.run() + + +def test_extract_ids_button(launch): + """Ensure clicking 'Extract IDs' triggers process and UI updates accordingly.""" + launch.run(timeout=10) + time.sleep(3) + + # Ensure 'Extract ids' button exists + extract_button = next((btn for btn in launch.button if "Extract ids" in btn.label), None) + assert extract_button is not None, "Extract ids button not found!" + + # Click the 'Extract ids' button + extract_button.click() + launch.run(timeout=10) + + print("Extract ids button was clicked successfully!") \ No newline at end of file diff --git a/tests/test_simple_workflow.py b/tests/test_simple_workflow.py new file mode 100644 index 0000000..5a94c41 --- /dev/null +++ b/tests/test_simple_workflow.py @@ -0,0 +1,69 @@ +import pytest +import time +from streamlit.testing.v1 import AppTest + +""" +Tests for the Simple Workflow page functionality. + +These tests verify: +- Number input widgets function correctly +- Session state updates properly +- Table generation with correct dimensions +- Download button presence +""" + +@pytest.fixture +def launch(): + """Launch the Simple Workflow page for testing.""" + app = AppTest.from_file("content/simple_workflow.py") + app.run(timeout=15) + return app + +def test_number_inputs(launch): + """Ensure x and y dimension inputs exist and update correctly.""" + + assert len(launch.number_input) >= 2, f"Expected at least 2 number inputs, found {len(launch.number_input)}" + + # Set x and y dimensions + x_input = next((ni for ni in launch.number_input if ni.key == "example-x-dimension"), None) + y_input = next((ni for ni in launch.number_input if ni.key == "example-y-dimension"), None) + + assert x_input is not None, "X-dimension input not found!" + assert y_input is not None, "Y-dimension input not found!" + + x_input.set_value(5) + y_input.set_value(4) + launch.run(timeout=10) + + # Validate session state updates + assert "example-x-dimension" in launch.session_state, "X-dimension key missing in session state!" + assert "example-y-dimension" in launch.session_state, "Y-dimension key missing in session state!" + assert launch.session_state["example-x-dimension"] == 5, "X-dimension not updated!" + assert launch.session_state["example-y-dimension"] == 4, "Y-dimension not updated!" + + assert len(launch.dataframe) > 0, "Table not generated!" + + df = launch.dataframe[0].value + assert df.shape == (5, 4), f"Expected table size (5,4) but got {df.shape}" + +def test_download_button(launch): + """Ensure 'Download Table' button appears after table generation.""" + + # Locate number inputs by key + x_input = next((ni for ni in launch.number_input if ni.key == "example-x-dimension"), None) + y_input = next((ni for ni in launch.number_input if ni.key == "example-y-dimension"), None) + + assert x_input is not None, "X-dimension input not found!" + assert y_input is not None, "Y-dimension input not found!" + + # Set values and trigger app update + x_input.set_value(3) + y_input.set_value(2) + launch.run(timeout=15) + time.sleep(5) + + assert len(launch.dataframe) > 0, "Table not generated!" + + # Find the "Download Table" button correctly + download_elements = [comp for comp in launch.main if hasattr(comp, "label") and "Download" in comp.label] + assert len(download_elements) > 0, "Download Table button is missing!" diff --git a/tests/test_tool_instance_name.py b/tests/test_tool_instance_name.py new file mode 100644 index 0000000..cd060ca --- /dev/null +++ b/tests/test_tool_instance_name.py @@ -0,0 +1,268 @@ +""" +Tests for the tool_instance_name functionality. + +This module verifies that save_parameters correctly resolves tool instance names +to real tool names when calling create_ini, and that parameters are stored and +retrieved using the instance name as the key. +""" +import os +import sys +import json +import pytest +import tempfile +from pathlib import Path +from unittest.mock import patch, MagicMock, call + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# Mock streamlit before importing ParameterManager +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +_original_streamlit = sys.modules.get('streamlit') +sys.modules['streamlit'] = mock_streamlit + +from src.workflow.ParameterManager import ParameterManager + +if _original_streamlit is not None: + sys.modules['streamlit'] = _original_streamlit +else: + sys.modules.pop('streamlit', None) + +# Remove cached src.workflow modules +for _key in list(sys.modules.keys()): + if _key.startswith('src.workflow'): + sys.modules.pop(_key, None) + + +@pytest.fixture +def temp_workflow_dir(): + """Create a temporary workflow directory for testing.""" + with tempfile.TemporaryDirectory() as tmpdir: + workflow_dir = Path(tmpdir) / "test-workflow" + workflow_dir.mkdir() + ini_dir = workflow_dir / "ini" + ini_dir.mkdir() + yield workflow_dir + + +@pytest.fixture(autouse=True) +def reset_streamlit_state(): + """Reset mock streamlit session state before each test.""" + mock_streamlit.session_state.clear() + yield + + +class TestSaveParametersWithInstanceName: + """Tests for save_parameters correctly resolving tool instance names.""" + + def test_save_parameters_uses_real_tool_name_for_create_ini(self, temp_workflow_dir): + """Test that save_parameters resolves instance name to real tool name + before calling create_ini.""" + pm = ParameterManager(temp_workflow_dir) + + # Simulate session state with an instance name (IDFilter_step1) + # that differs from the real tool name (IDFilter) + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + # Register instance mapping (as input_TOPP would) + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + + # Mock create_ini to track calls - return False (tool not found) + # to prevent further processing that requires actual ini files + with patch.object(pm, 'create_ini', return_value=False) as mock_create_ini: + pm.save_parameters() + + # Verify create_ini was called with the REAL tool name, not the instance name + mock_create_ini.assert_called_once_with("IDFilter") + + def test_save_parameters_without_instance_map_uses_tool_name_directly(self, temp_workflow_dir): + """Test that save_parameters works normally when no instance map exists + (backward compatibility).""" + pm = ParameterManager(temp_workflow_dir) + + # Simulate session state with a normal tool name (no instance mapping) + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter:1:score:pep"] = 0.05 + + with patch.object(pm, 'create_ini', return_value=False) as mock_create_ini: + pm.save_parameters() + + # Should use the tool name directly + mock_create_ini.assert_called_once_with("IDFilter") + + def test_save_parameters_stores_under_instance_name(self, temp_workflow_dir): + """Test that parameters are stored in JSON under the instance name, not + the real tool name.""" + pm = ParameterManager(temp_workflow_dir) + + # Create a mock ini file for IDFilter + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + # Set up instance mapping + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + + # Mock pyopenms Param and ParamXMLFile to avoid needing real ini files + mock_param = MagicMock() + mock_param.getValue.return_value = 0.01 # Different from session state value + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile') as mock_xml: + pm.save_parameters() + + # Load saved parameters + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # Parameters should be stored under the instance name + assert "IDFilter_step1" in saved + assert saved["IDFilter_step1"]["score:pep"] == 0.05 + + def test_save_parameters_multiple_instances_same_tool(self, temp_workflow_dir): + """Test that two instances of the same tool get separate parameter entries.""" + pm = ParameterManager(temp_workflow_dir) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + # Set up two instances with different parameter values + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter", + "IDFilter_step2": "IDFilter", + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.01 + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step2:1:score:pep"] = 0.05 + + mock_param = MagicMock() + mock_param.getValue.return_value = 0.0 # Default differs from both + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # Both instances should have separate entries + assert "IDFilter_step1" in saved + assert "IDFilter_step2" in saved + assert saved["IDFilter_step1"]["score:pep"] == 0.01 + assert saved["IDFilter_step2"]["score:pep"] == 0.05 + + def test_save_parameters_ini_key_maps_instance_to_real_tool(self, temp_workflow_dir): + """Test that ini_key correctly maps instance name back to real tool name + for param.getValue lookup.""" + pm = ParameterManager(temp_workflow_dir) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + + mock_param = MagicMock() + mock_param.getValue.return_value = 0.01 + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + pm.save_parameters() + + # Verify that param.getValue was called with the REAL tool name key + # (IDFilter:1:score:pep), not the instance name key (IDFilter_step1:1:score:pep) + mock_param.getValue.assert_called_with(b"IDFilter:1:score:pep") + + def test_save_parameters_display_keys_skipped_with_instance_name(self, temp_workflow_dir): + """Test that _display keys are still skipped when using instance names.""" + pm = ParameterManager(temp_workflow_dir) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep_display"] = ["0.05"] + + mock_param = MagicMock() + mock_param.getValue.return_value = 0.01 + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # _display key should not be stored + assert "score:pep_display" not in saved.get("IDFilter_step1", {}) + assert "score:pep" in saved.get("IDFilter_step1", {}) + + +class TestGetToppParametersWithInstanceName: + """Tests for get_topp_parameters with tool_instance_name.""" + + def test_get_topp_parameters_with_instance_name(self, temp_workflow_dir): + """Test that get_topp_parameters uses instance name for JSON lookup.""" + pm = ParameterManager(temp_workflow_dir) + + # Create params.json with instance-keyed parameters + params = { + "IDFilter_step1": { + "score:pep": 0.05 + } + } + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Create a mock ini file + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_param = MagicMock() + mock_param.keys.return_value = [b"IDFilter:1:score:pep"] + mock_param.getValue.return_value = 0.01 # default + + with patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + result = pm.get_topp_parameters("IDFilter", tool_instance_name="IDFilter_step1") + + # Should return the instance-specific value + assert result["score:pep"] == 0.05 + + def test_get_topp_parameters_without_instance_name_backward_compat(self, temp_workflow_dir): + """Test that get_topp_parameters works without instance name (backward compat).""" + pm = ParameterManager(temp_workflow_dir) + + params = { + "IDFilter": { + "score:pep": 0.05 + } + } + with open(pm.params_file, "w") as f: + json.dump(params, f) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_param = MagicMock() + mock_param.keys.return_value = [b"IDFilter:1:score:pep"] + mock_param.getValue.return_value = 0.01 + + with patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + result = pm.get_topp_parameters("IDFilter") + + assert result["score:pep"] == 0.05 diff --git a/tests/test_topp_flag_parameters.py b/tests/test_topp_flag_parameters.py new file mode 100644 index 0000000..627f50e --- /dev/null +++ b/tests/test_topp_flag_parameters.py @@ -0,0 +1,346 @@ +""" +Unit tests for PR #397 β€” flag parameter support for TOPP tools. + +PR #397 lets a caller mark certain TOPP parameters as CLI *flags*: parameters +passed by presence only (e.g. ``-force``), without a trailing value. The flag +names are persisted per tool instance by ``input_TOPP()`` into both +``st.session_state["_topp_flag_params"]`` and ``params.json["_flag_params"]``, +and consumed by ``run_topp()`` in ``src/workflow/CommandExecutor.py`` when it +builds the command line. + +These tests exercise ``run_topp()`` β€” the consumer that turns the persisted flag +definitions and merged parameters into an actual command. Driving ``run_topp()`` +also validates the persistence *contract* (the exact ``_flag_params`` / +``_topp_flag_params`` shapes that ``input_TOPP()`` writes), which is where the two +halves of the feature meet. + +The suite covers the working behaviour of the feature and also guards the two +issues CodeRabbit flagged during review, which are now fixed in ``run_topp()``: + - Finding 1 (per-tool flag fallback): + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023551 + - Finding 2 (list expansion / empty-list skipping): + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 +The tests that pin those findings carry a "CodeRabbit finding N" note in their +docstrings and live alongside the related behaviour they protect. +""" +import os +import sys +import json +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# --------------------------------------------------------------------------- +# Import the modules under test with `streamlit` (and `pyopenms`) mocked at the +# sys.modules level. Both CommandExecutor and ParameterManager do +# `import streamlit as st` at module top (ParameterManager also +# `import pyopenms as poms`). run_topp() only needs `st.session_state` to behave +# like a plain dict and never touches pyopenms, so lightweight mocks keep the +# test runnable without those heavy deps installed while still exercising the +# real command-construction logic. Mirrors tests/test_tool_instance_name.py. +# --------------------------------------------------------------------------- +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +_original_streamlit = sys.modules.get("streamlit") +_original_pyopenms = sys.modules.get("pyopenms") +sys.modules["streamlit"] = mock_streamlit +if _original_pyopenms is None: + sys.modules["pyopenms"] = MagicMock() + +from src.workflow.ParameterManager import ParameterManager +from src.workflow.CommandExecutor import CommandExecutor + +# Restore the original modules so other test files import the real ones. The +# classes imported above keep their module-level `st`/`poms` bound to the mocks. +if _original_streamlit is not None: + sys.modules["streamlit"] = _original_streamlit +else: + sys.modules.pop("streamlit", None) +if _original_pyopenms is None: + sys.modules.pop("pyopenms", None) + +for _key in list(sys.modules.keys()): + if _key.startswith("src.workflow"): + sys.modules.pop(_key, None) + + +TOOL = "FeatureFinderMetabo" + + +@pytest.fixture(autouse=True) +def reset_session_state(): + """Give each test a fresh, empty mocked session_state.""" + mock_streamlit.session_state = {} + yield + mock_streamlit.session_state = {} + + +def build_command( + params_json=None, + session_state=None, + *, + tool=TOOL, + input_output=None, + custom_params=None, + tool_instance_name=None, +): + """ + Invoke ``run_topp()`` with the supplied ``params.json`` content and + ``session_state``, and return the single command list it builds. + + ``run_command`` / ``run_multiple_commands`` are stubbed so nothing is + executed; the built command is captured from the ``run_command`` mock. + ``max_threads`` is pinned to 1 so the trailing ``-threads`` argument is + deterministic. + """ + if input_output is None: + input_output = {"in": ["input.mzML"], "out": ["output.featureXML"]} + + params_json = dict(params_json or {}) + params_json.setdefault("max_threads", 1) + + with tempfile.TemporaryDirectory() as tmpdir: + workflow_dir = Path(tmpdir) + pm = ParameterManager(workflow_dir) + with open(pm.params_file, "w", encoding="utf-8") as f: + json.dump(params_json, f) + + mock_streamlit.session_state = dict(session_state or {}) + + executor = CommandExecutor(workflow_dir, MagicMock(), pm) + executor.run_command = MagicMock(return_value=True) + executor.run_multiple_commands = MagicMock(return_value=True) + + executor.run_topp( + tool, + input_output, + custom_params=custom_params or {}, + tool_instance_name=tool_instance_name or tool, + ) + + assert executor.run_command.call_count == 1, ( + "expected exactly one single-process command, got " + f"{executor.run_command.call_count}" + ) + return executor.run_command.call_args.args[0] + + +# --------------------------- assertion helpers ----------------------------- + +def has_flag(cmd, name): + """True if ``-name`` appears anywhere in the command.""" + return f"-{name}" in cmd + + +def token_after(cmd, name): + """The single token immediately following ``-name`` (or None if it is last).""" + idx = cmd.index(f"-{name}") + return cmd[idx + 1] if idx + 1 < len(cmd) else None + + +def values_after(cmd, name): + """All value tokens following ``-name`` up to the next ``-flag`` token.""" + idx = cmd.index(f"-{name}") + vals = [] + for tok in cmd[idx + 1:]: + if tok.startswith("-"): + break + vals.append(tok) + return vals + + +def is_bare_flag(cmd, name): + """True if ``-name`` is present with no value (next token is another flag).""" + if not has_flag(cmd, name): + return False + nxt = token_after(cmd, name) + return nxt is None or nxt.startswith("-") + + +# ============================ working behaviour ============================ + + +class TestCommandSkeleton: + def test_input_output_files_prefixed(self): + cmd = build_command() + assert cmd[0] == TOOL + assert cmd[1:5] == ["-in", "input.mzML", "-out", "output.featureXML"] + # threads pinned to 1 and always appended last + assert cmd[-2:] == ["-threads", "1"] + + def test_collected_files_passed_as_single_list(self): + # A [["a", "b"]] entry is expanded in place after its -key. + cmd = build_command(input_output={"in": [["a.mzML", "b.mzML"]], "out": ["c.featureXML"]}) + assert cmd[1:4] == ["-in", "a.mzML", "b.mzML"] + + +class TestFlagParameters: + """Flags emit a bare ``-key`` when enabled and nothing when disabled.""" + + def test_flag_true_bool_emits_bare_flag(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": True}} + ) + assert is_bare_flag(cmd, "force") + assert "True" not in cmd + + def test_flag_string_true_emits_bare_flag(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": "true"}} + ) + assert is_bare_flag(cmd, "force") + assert "true" not in cmd + + def test_flag_string_true_is_case_insensitive(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": "True"}} + ) + assert is_bare_flag(cmd, "force") + + def test_flag_false_bool_omitted(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": False}} + ) + assert not has_flag(cmd, "force") + + def test_flag_string_false_omitted(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": "false"}} + ) + assert not has_flag(cmd, "force") + + +class TestRegularParameters: + """Non-flag merged parameters keep the existing value-appending behaviour.""" + + def test_empty_string_skipped(self): + cmd = build_command({TOOL: {"opt": ""}}) + assert not has_flag(cmd, "opt") + + def test_none_skipped(self): + cmd = build_command({TOOL: {"opt": None}}) + assert not has_flag(cmd, "opt") + + def test_zero_is_preserved(self): + # 0 and 0.0 are valid values, not "empty" β€” they must be passed through. + cmd = build_command({TOOL: {"min_int": 0, "min_float": 0.0}}) + assert values_after(cmd, "min_int") == ["0"] + assert values_after(cmd, "min_float") == ["0.0"] + + def test_scalar_value_appended(self): + cmd = build_command({TOOL: {"mz_tolerance": 10.5}}) + assert values_after(cmd, "mz_tolerance") == ["10.5"] + + def test_multiline_string_split_into_args(self): + cmd = build_command({TOOL: {"seq": "ALPHA\nBETA\nGAMMA"}}) + assert values_after(cmd, "seq") == ["ALPHA", "BETA", "GAMMA"] + + def test_merged_list_param_expanded(self): + """ + CodeRabbit finding 2 (fixed): a list-valued merged parameter expands into + separate CLI args, not its Python ``str()`` (e.g. ``"['a', 'b']"``). + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 + """ + cmd = build_command({TOOL: {"ids": ["a", "b"]}}) + assert values_after(cmd, "ids") == ["a", "b"] + + def test_merged_empty_list_param_skipped(self): + """ + CodeRabbit finding 2 (fixed): an empty-list merged parameter is omitted + entirely rather than emitting a ``-key`` with no usable value. + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 + """ + cmd = build_command({TOOL: {"ids": []}}) + assert not has_flag(cmd, "ids") + + +class TestCustomParameters: + """custom_params share the flag set and expand non-empty lists.""" + + def test_custom_flag_truthy_bare(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}}, + custom_params={"force": True}, + ) + assert is_bare_flag(cmd, "force") + + def test_custom_flag_false_omitted(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}}, + custom_params={"force": False}, + ) + assert not has_flag(cmd, "force") + + def test_custom_scalar_value(self): + cmd = build_command(custom_params={"extra": 5}) + assert values_after(cmd, "extra") == ["5"] + + def test_custom_nonempty_list_expanded(self): + cmd = build_command(custom_params={"ids": ["a", "b", "c"]}) + assert values_after(cmd, "ids") == ["a", "b", "c"] + + def test_custom_empty_string_skipped(self): + cmd = build_command(custom_params={"opt": ""}) + assert not has_flag(cmd, "opt") + + def test_custom_empty_list_param_skipped(self): + """ + CodeRabbit finding 2 (fixed): an empty-list custom parameter is omitted + rather than emitting a bare ``-key`` with no value. + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 + """ + cmd = build_command(custom_params={"ids": []}) + assert not has_flag(cmd, "ids") + + +class TestFlagSourceContract: + """Where run_topp() reads the flag definitions from.""" + + def test_flag_params_loaded_from_params_json(self): + # Survives a session restart: only params.json carries the flag list. + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": True}}, + session_state={}, + ) + assert is_bare_flag(cmd, "force") + + def test_fallback_to_session_state_when_json_has_no_flags(self): + # params.json has no _flag_params at all -> live session_state is used. + cmd = build_command( + {TOOL: {"force": True}}, + session_state={"_topp_flag_params": {TOOL: ["force"]}}, + ) + assert is_bare_flag(cmd, "force") + + def test_params_json_takes_priority_over_session_state(self): + # params.json says "force" is a flag; session_state disagrees (empty). + # params.json wins, so force is treated as a flag (bare, no value). + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": True}}, + session_state={"_topp_flag_params": {TOOL: []}}, + ) + assert is_bare_flag(cmd, "force") + assert "True" not in cmd + + def test_flag_fallback_uses_current_tool_when_other_tool_has_flags(self): + """ + CodeRabbit finding 1 (fixed): when params.json._flag_params holds an entry + for a DIFFERENT tool, the current tool's flags must still be read from the + session_state fallback. Previously the global ``if not flag_map`` check + skipped the fallback whenever any tool had flags, so the current tool's + flag was treated as a regular parameter and emitted as ``-force True``. + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023551 + """ + cmd = build_command( + {"_flag_params": {"OtherTool": ["some_flag"]}, TOOL: {"force": True}}, + session_state={"_topp_flag_params": {TOOL: ["force"]}}, + ) + assert is_bare_flag(cmd, "force") + assert "True" not in cmd From f61f3f19eb634df06d589c137cacf6395b435155 Mon Sep 17 00:00:00 2001 From: Yoo HoJun Date: Thu, 13 Aug 2026 14:58:34 +0900 Subject: [PATCH 09/10] Replace default-parameters.json, presets.json, settings.json with streamlit-template values Per follow-up instruction: these three config files now also take the template's values (app-name reverts to "OpenMS WebApp Template", library generation defaults are gone, presets/settings match template's set). Co-Authored-By: Claude Sonnet 5 --- default-parameters.json | 13 ++++++----- presets.json | 49 +++++++++++++++++++++++++++++++---------- settings.json | 10 +++++++-- 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/default-parameters.json b/default-parameters.json index 11479a2..7c084f8 100644 --- a/default-parameters.json +++ b/default-parameters.json @@ -1,9 +1,10 @@ { + "example-workflow-selected-mzML-files": [], "image-format": "svg", - "controllo": false, - "generate-library": false, - "library-use-fdr": false, - "library-psm-fdr": 0.01, - "library-generate-decoys": true, - "library-decoy-method": "shuffle" + "2D-map-intensity-cutoff": 5000, + + "example-x-dimension": 10, + "example-y-dimension": 5, + + "controllo": false } diff --git a/presets.json b/presets.json index d1878af..1099493 100644 --- a/presets.json +++ b/presets.json @@ -1,19 +1,44 @@ { "topp-workflow": { - "High Res MS": { - "_description": "Optimized for high-resolution MS2 (Orbitrap, TOF) when using ppm fragment tolerance", - "CometAdapter": { - "instrument": "high_res", - "fragment_mass_tolerance": 0.015, - "fragment_bin_offset": 0.0 + "High Sensitivity": { + "_description": "Optimized for detecting low-abundance features with higher noise tolerance", + "FeatureFinderMetabo": { + "algorithm:common:noise_threshold_int": 500.0, + "algorithm:common:chrom_peak_snr": 2.0, + "algorithm:mtd:mass_error_ppm": 15.0 } }, - "Low Res MS": { - "_description": "Optimized for low-resolution MS2 (Ion trap) when using ppm fragment tolerance", - "CometAdapter": { - "instrument": "low_res", - "fragment_mass_tolerance": 0.50025, - "fragment_bin_offset": 0.4 + "High Specificity": { + "_description": "Strict parameters for high-confidence feature detection", + "FeatureFinderMetabo": { + "algorithm:common:noise_threshold_int": 5000.0, + "algorithm:common:chrom_peak_snr": 5.0, + "algorithm:mtd:mass_error_ppm": 5.0 + } + }, + "Fast Analysis": { + "_description": "Faster processing with relaxed parameters for quick exploration", + "FeatureFinderMetabo": { + "algorithm:common:noise_threshold_int": 2000.0, + "algorithm:ffm:isotope_filtering_model": "none" + }, + "FeatureLinkerUnlabeledKD": { + "algorithm:link:rt_tol": 60.0, + "algorithm:link:mz_tol": 15.0 + } + }, + "Metabolomics Default": { + "_description": "Balanced parameters for general metabolomics analysis", + "FeatureFinderMetabo": { + "algorithm:common:noise_threshold_int": 1000.0, + "algorithm:common:chrom_peak_snr": 3.0, + "algorithm:mtd:mass_error_ppm": 10.0, + "algorithm:ffm:charge_lower_bound": 1, + "algorithm:ffm:charge_upper_bound": 3 + }, + "FeatureLinkerUnlabeledKD": { + "algorithm:link:rt_tol": 30.0, + "algorithm:link:mz_tol": 10.0 } } } diff --git a/settings.json b/settings.json index f792ddc..60424cb 100644 --- a/settings.json +++ b/settings.json @@ -1,8 +1,13 @@ { - "app-name": "quantms-web (DDA-LFQ)", + "app-name": "OpenMS WebApp Template", "github-user": "OpenMS", - "version": "1.0", + "version": "1.1.1", "repository-name": "streamlit-template", + "legal_links": { + "impressum": "https://openms.de/impressum", + "privacy": "https://openms.de/privacy", + "terms": "https://openms.de/terms" + }, "analytics": { "google-analytics": { "enabled": false, @@ -22,6 +27,7 @@ "enable_workspaces": true, "test": true, "workspaces_dir": "..", + "local_data_dir": "", "queue_settings": { "default_timeout": 7200, "result_ttl": 86400 From c4b4ca20348af3b60f38f26a5c1e879140b5fc7e Mon Sep 17 00:00:00 2001 From: hjn0415a <174866446+hjn0415a@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:03:29 +0900 Subject: [PATCH 10/10] Restore quantms-web-specific config/docs and pin openms-insight to 0.2.0 The streamlit-template sync in the prior two commits replaced these along with everything else, but they should stay quantms-web-specific rather than take the template's generic versions: - .claude, .github, .streamlit (CI workflows, Claude skills, Streamlit config) - README.md (project-specific docs) - presets.json, default-parameters.json (quantms-web workflow defaults) Also pin openms-insight to the exact version (0.2.0) this app was verified against, instead of the open-ended >=0.1.13 range. Drop pr-397.patch (stray local artifact, not meant to be tracked) and add it to .gitignore. Co-Authored-By: Claude Sonnet 5 --- .claude/skills/create-workflow.md | 22 - .github/workflows/build-and-test.yml | 570 +----------------- .../build-windows-executable-app.yaml | 8 +- .github/workflows/ci.yml | 6 +- .github/workflows/ghcr-cleanup.yml | 26 - .../workflows/test-win-exe-w-embed-py.yaml | 44 +- .../workflows/test-win-exe-w-pyinstaller.yaml | 5 +- .github/workflows/workflow-tests.yml | 28 - .gitignore | 3 +- .streamlit/config.toml | 3 +- README.md | 230 ++----- default-parameters.json | 13 +- pr-397.patch | 119 ---- presets.json | 49 +- requirements.txt | 2 +- 15 files changed, 125 insertions(+), 1003 deletions(-) delete mode 100644 .github/workflows/workflow-tests.yml delete mode 100644 pr-397.patch diff --git a/.claude/skills/create-workflow.md b/.claude/skills/create-workflow.md index 38e981d..07856db 100644 --- a/.claude/skills/create-workflow.md +++ b/.claude/skills/create-workflow.md @@ -120,31 +120,10 @@ The 4 pages call these methods respectively: - `self.ui.input_TOPP("ToolName", custom_defaults={})` β€” auto-generated TOPP parameter UI - `self.ui.input_python("script_name")` β€” auto-generated Python tool parameter UI - `self.ui.input_widget(key, default, label)` β€” single custom widget -- `select_input_file`, `input_TOPP` and `input_widget` accept `reactive=True` to rerun `configure()` when the widget changes (for conditional UI β€” see below) ### Logging - `self.logger.log("message")` β€” log progress during execution -## Conditional UI (reactive) - -Parameter widgets are isolated in an `st.fragment` by default, so a change reruns only that -widget and can't show/hide other widgets. Pass `reactive=True` to render the widget in the -parent scope instead β€” a change then reruns `configure()`. Read the live value from -`st.session_state` (not `self.params`, which is stale within the rerun) using -`self.parameter_manager.param_prefix` for custom-widget keys or `topp_param_prefix` for TOPP -keys of the form `":1:"`. - -```python -@st.fragment -def configure(self) -> None: - pm = self.parameter_manager - # changing the tool's `type` selectbox reruns configure() - self.ui.input_TOPP("IsobaricAnalyzer", reactive=True) - iso_type = st.session_state.get(f"{pm.topp_param_prefix}IsobaricAnalyzer:1:type", "") - if iso_type.startswith("tmt"): - self.ui.input_widget("tmt-channels", 10, "TMT channels", widget_type="number") -``` - ## Reference Files - Example workflow: `src/Workflow.py` @@ -165,7 +144,6 @@ def configure(self) -> None: - [ ] `__init__` calls `super().__init__("Name", st.session_state["workspace"])` - [ ] `upload()`, `configure()`, `execution()`, `results()` implemented - [ ] `@st.fragment` on `configure()` and `results()` -- [ ] `reactive=True` on any widget whose value controls other widgets' visibility - [ ] 4 content pages created in `content/` - [ ] All 4 pages registered as a group in `app.py` - [ ] Default parameters added to `default-parameters.json` if needed diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 3b4055b..7dfeb75 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -38,10 +38,7 @@ jobs: kubectl kustomize k8s/overlays/prod/ | \ kubeconform -summary -strict -kubernetes-version 1.28.0 -skip IngressRoute - build-amd64: - # amd64 path. Produces per-arch tags `--amd64`; the - # multi-arch manifest under `-` (and `latest`) is stitched - # together in `create-manifest` once the sibling `build-arm64` succeeds. + build: needs: lint-manifests runs-on: ubuntu-latest permissions: @@ -53,8 +50,6 @@ jobs: include: - variant: full dockerfile: Dockerfile - - variant: simple - dockerfile: Dockerfile_simple steps: - uses: actions/checkout@v4 @@ -78,491 +73,60 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | - type=ref,event=branch,suffix=-${{ matrix.variant }}-amd64 - type=ref,event=tag,suffix=-${{ matrix.variant }}-amd64 - type=sha,prefix=,suffix=-${{ matrix.variant }}-amd64 - type=raw,value=latest-amd64,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - - - name: Build and conditionally push - uses: docker/build-push-action@v5 - with: - context: . - file: ${{ matrix.dockerfile }} - platforms: linux/amd64 - load: true - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - # provenance/attestations turn the pushed tag into a manifest list, - # which the create-manifest job's `docker manifest create` then - # refuses ("is a manifest list"). Keep the push as a single-platform - # image manifest β€” same as the build-arm64 job. - provenance: false - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }}-amd64 - cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-amd64,mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} - build-args: | - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - - - name: Retag for kind (image name the kustomize overlay points at) - run: | - # The prod overlay sets `newName: ghcr.io/openms/streamlit-template`, - # `newTag: main-full`. The rendered manifests reference that exact - # ref, so we need it loaded into kind under that name. Tag invariant - # across branches/variants so the test always works. - FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) - docker tag "$FIRST_TAG" ghcr.io/openms/streamlit-template:main-full - - - name: Save image as tar - run: docker save ghcr.io/openms/streamlit-template:main-full -o /tmp/image.tar - - - name: Upload image artifact - uses: actions/upload-artifact@v4 - with: - name: openms-streamlit-${{ matrix.variant }}-amd64-image - path: /tmp/image.tar - retention-days: 1 - - build-arm64: - # arm64 path. Runs on a native ARM64 runner (no QEMU). Produces per-arch - # tags `--arm64`; gets merged into the multi-arch manifest - # under `-` by the `create-manifest` job below. The build - # uses a separate `Dockerfile.arm` / `Dockerfile_simple.arm` that swaps - # the miniforge installer to aarch64 and (for the full variant) guards - # the THIRDPARTY/Linux/aarch64 copy. The built image is also uploaded as - # an artifact so the apptainer / nginx / traefik integration jobs can - # exercise the ARM image on a native ARM runner (matrix arch=arm64). - needs: lint-manifests - runs-on: ubuntu-24.04-arm - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - include: - - variant: full - dockerfile: Dockerfile.arm - - variant: simple - dockerfile: Dockerfile_simple.arm - steps: - - name: Free disk space - # OpenMS source build needs ~25 GB of scratch space; the ARM runner - # image is tighter than the AMD one out of the box. Mirrors what - # FLASHApp's publish-docker-images.yml does at the top of its ARM job. - run: | - # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl - # cache binaries there and fail if the directory is missing. - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true - sudo apt-get clean - df -h - - - uses: actions/checkout@v4 - - - name: Compute lowercase image name (OCI refs must be lowercase) - run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GHCR - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=branch,suffix=-${{ matrix.variant }}-arm64 - type=ref,event=tag,suffix=-${{ matrix.variant }}-arm64 - type=sha,prefix=,suffix=-${{ matrix.variant }}-arm64 - type=raw,value=latest-arm64,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + type=ref,event=branch,suffix=-${{ matrix.variant }} + type=ref,event=tag,suffix=-${{ matrix.variant }} + type=sha,prefix=,suffix=-${{ matrix.variant }} + type=raw,value=latest,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Build and conditionally push uses: docker/build-push-action@v5 with: context: . file: ${{ matrix.dockerfile }} - platforms: linux/arm64 load: true push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }}-arm64 - cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-arm64,mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} - provenance: false + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }} + cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2},mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} build-args: | GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - - name: Retag for kind (image name the kustomize overlay points at) + - name: Retag for kind (stable local tag) run: | - # The prod overlay sets `newName: ghcr.io/openms/streamlit-template`, - # `newTag: main-full`. The rendered manifests reference that exact - # ref, so we need it loaded into kind under that name. Tag invariant - # across branches/variants so the test always works. + # load:true above loaded all meta-action tags into local docker. + # Retag the first one to the stable name the kustomize overlay expects. FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) - docker tag "$FIRST_TAG" ghcr.io/openms/streamlit-template:main-full + docker tag "$FIRST_TAG" openms-streamlit:test - name: Save image as tar - run: docker save ghcr.io/openms/streamlit-template:main-full -o /tmp/image.tar + run: docker save openms-streamlit:test -o /tmp/image.tar - name: Upload image artifact uses: actions/upload-artifact@v4 with: - name: openms-streamlit-${{ matrix.variant }}-arm64-image + name: openms-streamlit-${{ matrix.variant }}-image path: /tmp/image.tar retention-days: 1 - create-manifest: - # Stitch the per-arch tags into multi-arch manifest lists. The manifest - # tags reuse the OLD scheme (`-`, `latest`) so existing - # consumers (k8s overlays, docker-compose users, `docker pull` callers) - # keep working transparently β€” docker now auto-selects the right arch - # on pull. PRs don't push per-arch tags, so there's nothing to merge. - needs: [build-amd64, build-arm64] - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - variant: [full, simple] - steps: - - name: Compute lowercase image name - run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" - - - name: Log in to GHCR - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Compute manifest tags - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # NB: no -amd64/-arm64 suffix here. These are the multi-arch - # manifest names; they must match the pre-arm64 tag scheme so - # `:main-full`, `:v1.0.0-full`, `:latest` continue to resolve. - tags: | - type=ref,event=branch,suffix=-${{ matrix.variant }} - type=ref,event=tag,suffix=-${{ matrix.variant }} - type=sha,prefix=,suffix=-${{ matrix.variant }} - type=raw,value=latest,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - - - name: Create and push multi-arch manifests - # Iterate over manifest tags (newline-separated from metadata-action) - # and merge the matching `-amd64` / `-arm64` per-arch tags into each. - # `--amend` makes the step idempotent across workflow_dispatch reruns. - # `docker manifest push` accepts only one ref per invocation, hence - # the loop. - run: | - set -euo pipefail - while IFS= read -r manifest_tag; do - [ -z "$manifest_tag" ] && continue - amd_tag="${manifest_tag}-amd64" - arm_tag="${manifest_tag}-arm64" - echo "Creating manifest ${manifest_tag} from:" - echo " amd: ${amd_tag}" - echo " arm: ${arm_tag}" - docker manifest create "$manifest_tag" \ - --amend "$amd_tag" \ - --amend "$arm_tag" - docker manifest push "$manifest_tag" - done <<< "${{ steps.meta.outputs.tags }}" - - test-apptainer: - # Apptainer/Singularity is the dominant container runtime on HPC clusters. - # It mounts the root filesystem read-only and runs as the host user's UID - # (not root inside the image). The entrypoint must tolerate both: this job - # exercises that contract by running the built image under apptainer and - # waiting for the streamlit /_stcore/health endpoint to come up. - # - # amd64 only: upstream apptainer does NOT publish arm64 .deb assets - # (https://github.com/apptainer/apptainer/releases β€” every release lists - # only `apptainer__amd64.deb`), so eWaterCycle/setup-apptainer fails - # on ubuntu-24.04-arm with "sudo exit code 100" when its - # `apt-get install ./apptainer_*.deb` resolves a non-existent package. - # Building apptainer from source on the arm runner would add ~15 min and - # significant maintenance surface for limited value (HPC SIF consumers - # remain amd64). Re-evaluate if upstream starts publishing arm64 builds. - needs: build-amd64 + test-nginx: + needs: build runs-on: ubuntu-latest strategy: fail-fast: false matrix: - variant: [full, simple] + variant: [full] steps: - uses: actions/checkout@v4 - - name: Free disk space - # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind - # node image plus loading the OCI tar into both docker and kind can - # exhaust it. The arm runner is even tighter. Same incantation as - # `build-arm64`'s "Free disk space" step. - run: | - # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl - # cache binaries there and fail if the directory is missing. - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true - sudo apt-get clean - df -h - - name: Download image artifact uses: actions/download-artifact@v4 with: - name: openms-streamlit-${{ matrix.variant }}-amd64-image + name: openms-streamlit-${{ matrix.variant }}-image path: /tmp - - name: Install apptainer - uses: eWaterCycle/setup-apptainer@v2 - with: - apptainer-version: 1.3.4 - - - name: Build SIF from docker-archive - run: | - sudo apptainer build /tmp/openms.sif docker-archive:///tmp/image.tar - sudo chmod a+r /tmp/openms.sif - - - name: Prepare host bind dirs (mountpoint contract) - run: | - # Host paths we'll bind into the SIF. Asserting writability through - # singularity's bind machinery requires that the destination paths - # exist as real directories in the squashfs (otherwise singularity - # silently degrades the bind to read-only via underlay). - mkdir -p /tmp/host-workspaces /tmp/host-mounted-data - echo "from-host-pretest" > /tmp/host-mounted-data/sentinel.txt - - - name: Start apptainer instance (read-only root, host UID, with binds) - run: | - # Default apptainer semantics: read-only root, no --writable-tmpfs. - # This matches how users on HPC clusters run the SIF. - # Use `instance run` (apptainer 1.1+), not `instance start`: the SIF - # was built from docker-archive, which populates %runscript with the - # Docker ENTRYPOINT but leaves %startscript as the default no-op - # `exec "$@"`. `instance start` would launch an empty instance and - # streamlit would never bind 8501. - apptainer instance run \ - --bind /tmp/host-workspaces:/workspaces-streamlit-template:rw \ - --bind /tmp/host-mounted-data:/mounted-data:ro \ - /tmp/openms.sif openms-test - apptainer instance list - # Record where this run's logs will land so subsequent steps can tail - # them deterministically (path depends on hostname/user). - LOG_DIR=$(find "$HOME/.apptainer/instances/logs" -type d -name "$(whoami)" 2>/dev/null | head -n 1) - echo "APPTAINER_LOG_DIR=${LOG_DIR}" >> "$GITHUB_ENV" - ls -la "$LOG_DIR" || true - - - name: Wait for streamlit /_stcore/health - run: | - # Tail the entrypoint's stdout/stderr alongside the health probe so - # any startup failure surfaces directly in the CI log (the dedicated - # "Dump entrypoint logs on failure" step is post-mortem only and - # easy to miss in the GH Actions UI). - OUT="${APPTAINER_LOG_DIR}/openms-test.out" - ERR="${APPTAINER_LOG_DIR}/openms-test.err" - for i in $(seq 1 90); do - if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8501/_stcore/health; then - echo "Streamlit is ready after $i attempts" - exit 0 - fi - if [ $((i % 5)) -eq 0 ]; then - echo "--- attempt $i: instance log tail ---" - tail -n 20 "$OUT" 2>/dev/null || echo "(no $OUT yet)" - tail -n 10 "$ERR" 2>/dev/null || echo "(no $ERR yet)" - apptainer instance list || true - fi - sleep 2 - done - echo "TIMED OUT waiting for streamlit health endpoint" - echo "--- full entrypoint stdout ---" - cat "$OUT" 2>/dev/null || echo "(missing)" - echo "--- full entrypoint stderr ---" - cat "$ERR" 2>/dev/null || echo "(missing)" - exit 1 - - - name: Verify health endpoint returns 200 - run: curl -fsS http://127.0.0.1:8501/_stcore/health - - - name: Verify Redis is reachable inside container (full variant) - if: matrix.variant == 'full' - run: | - # In apptainer mode the entrypoint uses a unix socket (TCP 6379 on - # localhost is the host's, since net namespace is shared). The - # entrypoint writes the resolved URL to /tmp/openms-redis-url for - # out-of-band discovery, since `apptainer exec` spawns a fresh - # shell that doesn't inherit the daemon's exported env. - URL=$(apptainer exec instance://openms-test cat /tmp/openms-redis-url 2>/dev/null || true) - case "$URL" in - unix://*) - SOCK="${URL#unix://}" - echo "Redis URL is unix socket: $SOCK" - apptainer exec instance://openms-test redis-cli -s "$SOCK" ping | grep -i pong - ;; - *) - echo "Redis URL is TCP (or unset): ${URL:-default}" - apptainer exec instance://openms-test redis-cli ping | grep -i pong - ;; - esac - - - name: Verify bind mount is writable (workspaces) and readable (data) - run: | - # The whole point of pre-creating /workspaces-streamlit-template - # and /mounted-data in the image: singularity now has a real - # attach point and `:rw` actually sticks. Without the mkdir, - # `apptainer exec ... touch` here would fail with EROFS. - apptainer exec instance://openms-test sh -c \ - 'echo from-container > /workspaces-streamlit-template/probe.txt' - test -f /tmp/host-workspaces/probe.txt - grep -q from-container /tmp/host-workspaces/probe.txt - # Read-only data mount should also be visible inside the container. - apptainer exec instance://openms-test grep -q from-host-pretest /mounted-data/sentinel.txt - # The mounted-drive browser uses os.path.ismount() to gate - # rendering (existence is no longer enough now that the image - # pre-creates the dir). Assert the kernel reports both paths as - # real mount points so the detection function returns truthy. - apptainer exec instance://openms-test python3 -c " - import os, sys - for p in ('/mounted-data', '/workspaces-streamlit-template'): - assert os.path.ismount(p), f'{p} not reported as mount point' - print(f'ismount({p}) = True') - " - - - name: Dump entrypoint logs on failure - if: failure() - run: | - echo "--- apptainer instance list ---" - apptainer instance list || true - echo "--- apptainer instance logs ---" - find "$HOME/.apptainer" \( -name '*.out' -o -name '*.err' \) 2>/dev/null \ - | while read -r f; do echo "=== $f ==="; cat "$f"; done || true - - - name: Stop apptainer instance - if: always() - run: apptainer instance stop openms-test || true - - - name: Upload validated SIF artifact (push events only) - if: success() && github.event_name != 'pull_request' - uses: actions/upload-artifact@v4 - with: - name: openms-streamlit-${{ matrix.variant }}-sif - path: /tmp/openms.sif - retention-days: 1 - if-no-files-found: error - - publish-apptainer: - # Publish the validated SIF (already health-checked above) to GHCR as an - # OCI artifact via ORAS, in a sibling package: ghcr.io///sif. - # Keeping it separate from the docker image package keeps tag lists clean - # and lets HPC users `apptainer pull oras://...` without the 5-15 min - # on-the-fly OCI->SIF conversion the docker:// path requires. - needs: test-apptainer - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - variant: [full, simple] - steps: - - name: Download validated SIF artifact - uses: actions/download-artifact@v4 - with: - name: openms-streamlit-${{ matrix.variant }}-sif - path: /tmp - - - name: Install apptainer - uses: eWaterCycle/setup-apptainer@v2 - with: - apptainer-version: 1.3.4 - - - name: Compute SIF tags - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/sif - tags: | - type=ref,event=branch,suffix=-${{ matrix.variant }} - type=ref,event=tag,suffix=-${{ matrix.variant }} - type=sha,prefix=,suffix=-${{ matrix.variant }} - type=raw,value=latest,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - - - name: Log in to GHCR for ORAS push - env: - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # apptainer reads its auth from ~/.apptainer/remote.yaml, NOT from - # ~/.docker/config.json β€” so docker/login-action won't work here. - # Login and push must both run as the runner user (no sudo) so they - # share the same $HOME and therefore the same auth file. - echo "$GHCR_TOKEN" | apptainer registry login \ - --username "${{ github.actor }}" \ - --password-stdin \ - oras://ghcr.io - - - name: Push SIF to each computed tag - run: | - # `apptainer push` accepts ONE destination per invocation; iterate - # over the newline-separated tag list from docker/metadata-action. - # tr lowercase is belt-and-braces β€” metadata-action already - # lowercases, but GHCR is strict about case in OCI refs. - set -euo pipefail - while IFS= read -r tag; do - [ -z "$tag" ] && continue - tag_lc="$(echo "$tag" | tr '[:upper:]' '[:lower:]')" - echo "Pushing SIF to oras://${tag_lc}" - apptainer push /tmp/openms.sif "oras://${tag_lc}" - done <<< "${{ steps.meta.outputs.tags }}" - - test-nginx: - needs: [build-amd64, build-arm64] - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - include: - - variant: full - arch: amd64 - runner: ubuntu-latest - - variant: full - arch: arm64 - runner: ubuntu-24.04-arm - - variant: simple - arch: amd64 - runner: ubuntu-latest - - variant: simple - arch: arm64 - runner: ubuntu-24.04-arm - steps: - - uses: actions/checkout@v4 - - - name: Free disk space - # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind - # node image plus loading the OCI tar into both docker and kind can - # exhaust it. The arm runner is even tighter. Same incantation as - # `build-arm64`'s "Free disk space" step. - run: | - # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl - # cache binaries there and fail if the directory is missing. - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true - sudo apt-get clean - df -h - - - name: Download image artifact - uses: actions/download-artifact@v4 - with: - name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image - path: /tmp + - name: Load image into local docker + run: docker load -i /tmp/image.tar - name: Create kind cluster uses: helm/kind-action@v1 @@ -571,13 +135,7 @@ jobs: config: .github/kind-config.yaml - name: Load image into kind cluster - # Use `kind load image-archive` (not docker-image) so we never store - # the image in host docker. Saves ~5-8 GB on /var/lib/docker. Delete - # the tar afterwards to free the same again on /tmp β€” the image is - # now in both kind nodes' containerd, which is enough. - run: | - kind load image-archive /tmp/image.tar --name test-cluster - rm -f /tmp/image.tar + run: kind load docker-image openms-streamlit:test --name test-cluster - name: Install nginx ingress controller run: | @@ -589,7 +147,7 @@ jobs: # Filter out Traefik IngressRoute (kind cluster uses nginx) and force imagePullPolicy=Never kubectl kustomize k8s/overlays/prod/ | \ yq 'select(.kind != "IngressRoute")' | \ - sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' | \ + sed 's|imagePullPolicy: IfNotPresent|imagePullPolicy: Never|g' | \ sed 's|storageClassName: cinder-csi|storageClassName: standard|g' > /tmp/manifests.yaml for i in 1 2 3 4 5; do if kubectl apply -f /tmp/manifests.yaml; then @@ -638,67 +196,25 @@ jobs: echo "$host -> 200 OK" done - - name: Dump cluster state on failure - if: failure() - run: | - echo "=== nodes ===" - kubectl get nodes -o wide || true - echo "=== pods (all namespaces) ===" - kubectl get pods -A -o wide || true - echo "=== app pods describe ===" - kubectl describe pod -n openms -l app=${SLUG} || true - echo "=== app pod logs ===" - kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix || true - echo "=== app pod previous logs (if crashed) ===" - kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix --previous || true - echo "=== ingress ===" - kubectl get ingress -A -o wide || true - kubectl describe ingress -n openms || true - echo "=== services + endpoints ===" - kubectl get svc,endpoints -n openms || true - echo "=== ingress-nginx controller logs ===" - kubectl logs -n ingress-nginx -l app.kubernetes.io/component=controller --tail=200 || true - test-traefik: - needs: [build-amd64, build-arm64] - runs-on: ${{ matrix.runner }} + needs: build + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - include: - - variant: full - arch: amd64 - runner: ubuntu-latest - - variant: full - arch: arm64 - runner: ubuntu-24.04-arm - - variant: simple - arch: amd64 - runner: ubuntu-latest - - variant: simple - arch: arm64 - runner: ubuntu-24.04-arm + variant: [full] steps: - uses: actions/checkout@v4 - - name: Free disk space - # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind - # node image plus loading the OCI tar into both docker and kind can - # exhaust it. The arm runner is even tighter. Same incantation as - # `build-arm64`'s "Free disk space" step. - run: | - # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl - # cache binaries there and fail if the directory is missing. - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true - sudo apt-get clean - df -h - - name: Download image artifact uses: actions/download-artifact@v4 with: - name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image + name: openms-streamlit-${{ matrix.variant }}-image path: /tmp + - name: Load image into local docker + run: docker load -i /tmp/image.tar + - name: Create kind cluster uses: helm/kind-action@v1 with: @@ -706,13 +222,7 @@ jobs: config: .github/kind-config.yaml - name: Load image into kind cluster - # Use `kind load image-archive` (not docker-image) so we never store - # the image in host docker. Saves ~5-8 GB on /var/lib/docker. Delete - # the tar afterwards to free the same again on /tmp β€” the image is - # now in both kind nodes' containerd, which is enough. - run: | - kind load image-archive /tmp/image.tar --name traefik-test - rm -f /tmp/image.tar + run: kind load docker-image openms-streamlit:test --name traefik-test - name: Set up Helm uses: azure/setup-helm@v4 @@ -729,7 +239,7 @@ jobs: - name: Deploy with Kustomize (full manifests, no filter) run: | kubectl kustomize k8s/overlays/prod/ | \ - sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' | \ + sed 's|imagePullPolicy: IfNotPresent|imagePullPolicy: Never|g' | \ sed 's|storageClassName: cinder-csi|storageClassName: standard|g' > /tmp/manifests.yaml for i in 1 2 3 4 5; do if kubectl apply -f /tmp/manifests.yaml; then @@ -777,23 +287,3 @@ jobs: echo "" echo "$host -> 200 OK" done - - - name: Dump cluster state on failure - if: failure() - run: | - echo "=== nodes ===" - kubectl get nodes -o wide || true - echo "=== pods (all namespaces) ===" - kubectl get pods -A -o wide || true - echo "=== app pods describe ===" - kubectl describe pod -n openms -l app=${SLUG} || true - echo "=== app pod logs ===" - kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix || true - echo "=== app pod previous logs (if crashed) ===" - kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix --previous || true - echo "=== traefik ingressroute ===" - kubectl get ingressroute -A -o yaml || true - echo "=== services + endpoints ===" - kubectl get svc,endpoints -n openms || true - echo "=== traefik controller logs ===" - kubectl logs -n traefik -l app.kubernetes.io/name=traefik --tail=200 || true diff --git a/.github/workflows/build-windows-executable-app.yaml b/.github/workflows/build-windows-executable-app.yaml index 99b4177..e1af54b 100644 --- a/.github/workflows/build-windows-executable-app.yaml +++ b/.github/workflows/build-windows-executable-app.yaml @@ -19,11 +19,11 @@ env: OPENMS_CONTRIB_VERSION: "" PYTHON_VERSION: 3.11.0 # Name of the installer - APP_NAME: OpenMS-StreamlitTemplateApp + APP_NAME: quantms-web-DDA-LFQ # Define unique GUID for UpgradeCode APP_UpgradeCode: "8d28e8c7-45dc-446c-b889-99a6aea2f1a5" # Define needed TOPP tools here - TOPP_TOOLS: "FeatureFinderMetabo FeatureLinkerUnlabeledKD SiriusExport" + TOPP_TOOLS: "DecoyDatabase CometAdapter PercolatorAdapter IDFilter ProteomicsLFQ" jobs: build-openms: @@ -76,7 +76,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ github.workspace }}/OpenMS/contrib - key: ${{ runner.os }}-contrib-${{ env.OPENMS_CONTRIB_VERSION || env.OPENMS_VERSION }} + key: ${{ runner.os }}-contrib-${{ env.OPENMS_VERSION }} - name: Load contrib build if: steps.cache-contrib-win.outputs.cache-hit != 'true' @@ -85,7 +85,7 @@ jobs: run: | cd OpenMS/contrib # Download the file using the URL fetched from GitHub - gh release download release/${{ env.OPENMS_CONTRIB_VERSION || env.OPENMS_VERSION }} -R OpenMS/contrib --pattern 'contrib_build-Windows.tar.gz' + gh release download release/${{ env.OPENMS_VERSION }} -R OpenMS/contrib --pattern 'contrib_build-Windows.tar.gz' # Extract the archive 7z x -so contrib_build-Windows.tar.gz | 7z x -si -ttar rm contrib_build-Windows.tar.gz diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34b5359..6d79f8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: continuous-integration -on: [push, pull_request] +on: [push] jobs: test: @@ -8,8 +8,8 @@ jobs: strategy: matrix: os: [ubuntu-latest] - # Match the python version used in the committed Dockerfile (release/3.5.0) - python-version: ["3.10"] + # Requirements file generated with python=3.12; tested with python=3.11 + python-version: ["3.11"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 diff --git a/.github/workflows/ghcr-cleanup.yml b/.github/workflows/ghcr-cleanup.yml index 00aca96..6228b62 100644 --- a/.github/workflows/ghcr-cleanup.yml +++ b/.github/workflows/ghcr-cleanup.yml @@ -51,29 +51,3 @@ jobs: tag-selection: untagged cut-off: 7d dry-run: ${{ github.event.inputs.dry-run || 'false' }} - - cleanup-sif-images: - runs-on: ubuntu-latest - permissions: - packages: write - steps: - - name: Delete old commit-tagged SIFs (keep semver + main + latest) - uses: snok/container-retention-policy@v3.0.1 - with: - account: ${{ github.repository_owner }} - token: ${{ secrets.GITHUB_TOKEN }} - image-names: ${{ github.event.repository.name }}/sif - image-tags: "!v*-full !v*-simple !main-full !main-simple !latest" - tag-selection: tagged - cut-off: 30d - dry-run: ${{ github.event.inputs.dry-run || 'false' }} - - - name: Delete untagged SIF manifests - uses: snok/container-retention-policy@v3.0.1 - with: - account: ${{ github.repository_owner }} - token: ${{ secrets.GITHUB_TOKEN }} - image-names: ${{ github.event.repository.name }}/sif - tag-selection: untagged - cut-off: 7d - dry-run: ${{ github.event.inputs.dry-run || 'false' }} diff --git a/.github/workflows/test-win-exe-w-embed-py.yaml b/.github/workflows/test-win-exe-w-embed-py.yaml index b543d0e..deec56d 100644 --- a/.github/workflows/test-win-exe-w-embed-py.yaml +++ b/.github/workflows/test-win-exe-w-embed-py.yaml @@ -1,5 +1,5 @@ name: Test streamlit executable for Windows with embeddable python -on: +on: push: branches: [ "main" ] workflow_dispatch: @@ -46,7 +46,7 @@ jobs: - name: Create .bat file run: | echo " start /min .\python-${{ env.PYTHON_VERSION }}\python -m streamlit run app.py local" > ${{ env.APP_NAME }}.bat - + - name: Create All-in-one executable folder run: | mkdir streamlit_exe @@ -86,20 +86,20 @@ jobs: - Join our Discord server for support and community discussions: https://discord.com/invite/4TAGhqJ7s5 - Contribute or stay updated with the latest OpenMS web app developments on GitHub: https://github.com/OpenMS/streamlit-template - Visit our website for more information: https://openms.de/ - + Thank you for using ${{ env.APP_NAME }}! EOF - + - name: Install WiX Toolset run: | curl -LO https://github.com/wixtoolset/wix3/releases/download/wix3111rtm/wix311-binaries.zip unzip wix311-binaries.zip -d wix rm wix311-binaries.zip - + - name: Build .wxs for streamlit_exe folder run: | ./wix/heat.exe dir streamlit_exe -gg -sfrag -sreg -srd -template component -cg StreamlitExeFiles -dr AppSubFolder -out streamlit_exe_files.wxs - + - name: Generate VBScript file shell: bash run: | @@ -115,7 +115,7 @@ jobs: cp assets/openms_license.rtf SourceDir # Logo of app cp assets/openms.ico SourceDir - + - name: Generate WiX XML file shell: bash run: | @@ -125,13 +125,13 @@ jobs: - + - + - + @@ -141,7 +141,7 @@ jobs: - + @@ -149,30 +149,30 @@ jobs: - + - + - + - + - + - + @@ -180,13 +180,13 @@ jobs: - + - - + + @@ -196,7 +196,7 @@ jobs: - name: Build .wixobj file with candle.exe run: | ./wix/candle.exe streamlit_exe.wxs streamlit_exe_files.wxs - + - name: Link .wixobj file into .msi with light.exe run: | ./wix/light.exe -ext WixUIExtension -sice:ICE60 -o ${{ env.APP_NAME }}.msi streamlit_exe_files.wixobj streamlit_exe.wixobj @@ -206,4 +206,4 @@ jobs: with: name: OpenMS-App-Test path: | - ${{ env.APP_NAME }}.msi \ No newline at end of file + ${{ env.APP_NAME }}.msi diff --git a/.github/workflows/test-win-exe-w-pyinstaller.yaml b/.github/workflows/test-win-exe-w-pyinstaller.yaml index 94c91ae..15e7ef4 100644 --- a/.github/workflows/test-win-exe-w-pyinstaller.yaml +++ b/.github/workflows/test-win-exe-w-pyinstaller.yaml @@ -19,10 +19,11 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Setup virtual environment - shell: cmd + shell: cmd run: | python -m venv myenv - call myenv\Scripts\activate.bat + call myenv\Scripts\activate.bat + pip install cython numpy pip install -r requirements.txt pip install pyinstaller diff --git a/.github/workflows/workflow-tests.yml b/.github/workflows/workflow-tests.yml deleted file mode 100644 index 92b0b99..0000000 --- a/.github/workflows/workflow-tests.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Test workflow functions - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pytest - - name: Running test cases - run: | - pytest test.py - - name: Running GUI tests - run: | - pytest test_gui.py diff --git a/.gitignore b/.gitignore index 1525129..7b228e0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ gdpr_consent/node_modules/ *~ .streamlit/secrets.toml docs/superpowers/ -.venv/ \ No newline at end of file +.venv/ +pr-397.patch \ No newline at end of file diff --git a/.streamlit/config.toml b/.streamlit/config.toml index b7f3cf6..9b41e52 100644 --- a/.streamlit/config.toml +++ b/.streamlit/config.toml @@ -10,8 +10,7 @@ developmentMode = false files = ["/app/admin-secrets/secrets.toml", "~/.streamlit/secrets.toml", ".streamlit/secrets.toml"] [server] -address = "0.0.0.0" -maxUploadSize = 200 #MB +maxUploadSize = 3000 #MB port = 8501 # should be same as configured in deployment repo diff --git a/README.md b/README.md index 5a1db4b..89f7d15 100644 --- a/README.md +++ b/README.md @@ -1,203 +1,55 @@ -# OpenMS streamlit template - -[![Open Template!](https://static.streamlit.io/badges/streamlit_badge_black_white.svg)](https://abi-services.cs.uni-tuebingen.de/streamlit-template/) - -This repository contains a template app for OpenMS workflows in a web application using the **streamlit** framework. It serves as a foundation for apps ranging from simple workflows with **pyOpenMS** to complex workflows utilizing **OpenMS TOPP tools** with parallel execution. It includes solutions for handling user data and parameters in workspaces as well as deployment with docker-compose. - -## Features - -- Workspaces for user data with unique shareable IDs -- Persistent parameters and input files within a workspace -- local and online mode -- Captcha control -- Packaged executables for Windows -- framework for workflows with OpenMS TOPP tools -- Deployment [with docker-compose](https://github.com/OpenMS/streamlit-deployment) - -## πŸ”— Try the Online Demo - -Explore the hosted version here: πŸ‘‰ [Live App](https://abi-services.cs.uni-tuebingen.de/streamlit-template/) - -## πŸ’» Run Locally - -To run the app locally: - -1. **Clone the repository** - ```bash - git clone https://github.com/OpenMS/streamlit-template.git - cd streamlit-template - ``` - -2. **Install dependencies** - - Make sure you can run ```pip``` commands. - - Install all dependencies with: - ```bash - pip install -r requirements.txt - ``` - -4. **Launch the app** - ```bash - streamlit run app.py - ``` - -> ⚠️ Note: The local version offers limited functionality. Features that depend on OpenMS TOPP tools are only available out of the box in the Docker setup. For the local version [OpenMS Command Line Tools](https://openms.readthedocs.io/en/latest/about/installation.html) must be installed separately. - - -## 🐳 Build with Docker - -This repository contains two Dockerfiles. - -1. `Dockerfile`: This Dockerfile builds all dependencies for the app including Python packages and the OpenMS TOPP tools. Recommended for more complex workflows where you want to use the OpenMS TOPP tools for instance with the **TOPP Workflow Framework**. -2. `Dockerfile_simple`: This Dockerfile builds only the Python packages. Recommended for simple apps using pyOpenMS only. - -1. **Install Docker** - - Install Docker from the [official Docker installation guide](https://docs.docker.com/engine/install/) - -
- Click to expand - - ```bash - # Remove older Docker versions (if any) - for pkg in docker.io docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do sudo apt-get remove -y $pkg; done - ``` - -
- -2. **Test Docker** - - Verify that Docker is working. - ```bash - docker run hello-world - ``` - When running this command, you should see a hello world message from Docker. - -3. **Clone the repository** - ```bash - git clone https://github.com/OpenMS/streamlit-template.git - cd streamlit-template - ``` - -4. **Specify GitHub token (to download Windows executables).** - - Create a temporary `.env` file with your Github token. - - It should contain only one line: - `GITHUB_TOKEN=` - - ℹ️ **Note:** This step is not strictly required, but skipping it will remove the option to download executables from the WebApp. - -3. **Build & Launch the App** - - To build and start the containers. - From the project root directory: - - ```bash - docker-compose up -d --build - ``` - At the end, you should see this: - ``` - [+] Running 2/2 - βœ” openms-streamlit-template Built - βœ” Container openms-streamlit-template Started - ``` - - To make sure server started successfully, run `docker compose ps`. You should see `Up` status: - ``` - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 4abe0603e521 openms_streamlit_template "/app/entrypoint.sh …" 7 minutes ago Up 7 minutes 0.0.0.0:8501->8501/tcp, :::8501->8501/tcp openms-streamlit-template - ``` - - To map the port to default streamlit port `8501` and launch. - - ``` - docker run -p 8505:8501 openms_streamlit_template - ``` - - ### Mount a local data directory - - To make a directory of MS files on the host available to the running app - without uploading or copying them, bind-mount it into the container at - the path configured by `local_data_dir` in `settings.json` (the Docker - image defaults this to `/mounted-data`): - - ``` - docker run -p 8501:8501 \ - -v /path/on/host:/mounted-data:ro \ - openms_streamlit_template - ``` - - The upload widget auto-detects the mount: when the directory exists at - runtime it shows an in-app tree browser; selected files are referenced - in place via `external_files.txt` (no copy into the workspace volume), - so the mount can safely be read-only. Omitting `-v` hides the browser - and falls back to the standard upload UI. To use a different container - path, change `local_data_dir` in `settings.json` before building. - -## πŸ›°οΈ Run with Apptainer / Singularity (HPC) - -Apptainer (formerly Singularity) is the dominant container runtime on HPC -clusters. CI publishes prebuilt SIFs to GHCR via ORAS, so you can pull a -ready-to-run image with no on-the-fly OCIβ†’SIF conversion and run it as your -user β€” no root, no `--writable-tmpfs` required: +# quantms-web β€” DDA Label-Free Quantification + +A browser-based **Data-Dependent Acquisition (DDA) Label-Free Quantification** workflow for proteomics. Upload mzML files and a protein FASTA; get identified and quantified proteins with volcano plots, PCA, and clustered heatmaps. No CLI, no Nextflow config. + +quantms-web mirrors the **dda-lfq branch of the [quantms Nextflow workflow](https://github.com/bigbio/quantms)** but runs as a [Streamlit](https://streamlit.io) app powered by OpenMS TOPP tools. + +## Pipeline + +| Stage | Tool | What it does | +|---|---|---| +| 1. Identification | Comet | Peptide-spectrum matching against a protein database | +| 2. Rescoring | Percolator | ML-based statistical validation of PSMs | +| 3. Filtering | IDFilter | FDR-controlled peptide identification filtering | +| 4. Quantification | ProteomicsLFQ | Label-free quantification across samples | +| 5. Analysis | Built-in | Volcano plots, PCA, heatmaps, spectral library export | + +## Run locally + +Install the Python dependencies and launch: ```bash -apptainer pull --name openms-streamlit-template.sif \ - oras://ghcr.io/openms/streamlit-template/sif:latest -apptainer run \ - --bind /path/to/data:/mounted-data:ro \ - --bind /path/to/workspaces:/workspaces-streamlit-template \ - openms-streamlit-template.sif +git clone https://github.com/OpenMS/quantms-web.git +cd quantms-web +pip install -r requirements.txt +streamlit run app.py ``` -Available tags follow the same scheme as the Docker images: `latest`, -`main-full`, `main-simple`, `v*-full`, `v*-simple`, and per-commit SHAs. -If a tag hasn't been prebuilt yet (e.g. a PR branch), fall back to on-the-fly -conversion: `apptainer pull docker://ghcr.io/openms/streamlit-template:`. -Requires apptainer 1.1+ or singularity-ce 3.10+ for the `oras://` transport. - -The entrypoint auto-detects the read-only root filesystem (set by apptainer's -default isolation) and switches its runtime state β€” Redis data directory, -nginx config, PID files β€” to `/tmp/openms-runtime-$$`, which is always -writable inside an apptainer container. The workspace cleanup cron job is -skipped in this mode; rerun `clean-up-workspaces.py` manually if needed. - -## βš–οΈ Legal pages (Impressum, Privacy Policy, Terms of Use) - -Every page shows **Impressum**, **Privacy Policy** and **Terms of Use** links at -the bottom of the sidebar, and the GDPR consent banner links to the privacy -policy. By default these point to the centrally maintained official OpenMS pages -(`https://openms.de/impressum`, `/privacy`, `/terms`). - -If you self-host a fork, override them in `settings.json` β€” an Impressum must -name the **actual operator**, not OpenMS: - -```json -"legal_links": { - "impressum": "https://your-domain.example/impressum", - "privacy": "https://your-domain.example/privacy", - "terms": "https://your-domain.example/terms" -} -``` +The full pipeline runs locally once the [OpenMS Command Line Tools](https://openms.readthedocs.io/en/latest/about/installation.html) are on your `PATH` β€” they provide Comet, Percolator, ProteomicsLFQ, and the rest of the TOPP suite. With Python alone, the pyOpenMS-backed parts of the UI still work. -Any link you omit falls back to its OpenMS default. The `privacy` URL is reused -for the consent banner's privacy-policy link, so consent and policy stay in sync. +## Run with Docker -## Documentation +Ships OpenMS and the search engines together so the full pipeline works out of the box: -Documentation for **users** and **developers** is included as pages in [this template app](https://abi-services.cs.uni-tuebingen.de/streamlit-template/), indicated by the πŸ“– icon. +```bash +docker-compose up -d --build +``` -## Citation +Open http://localhost:8501. -Please cite: -MΓΌller, T. D., Siraj, A., et al. OpenMS WebApps: Building User-Friendly Solutions for MS Analysis. Journal of Proteome Research (2025). [https://doi.org/10.1021/acs.jproteome.4c00872](https://doi.org/10.1021/acs.jproteome.4c00872) +## Windows installer -## References +Download the latest `.msi` from [Releases](https://github.com/OpenMS/quantms-web/releases) and double-click to install. Standalone β€” no Python or Docker required. -- Pfeuffer, J., Bielow, C., Wein, S. et al. OpenMS 3 enables reproducible analysis of large-scale mass spectrometry data. Nat Methods 21, 365–367 (2024). [https://doi.org/10.1038/s41592-024-02197-7](https://doi.org/10.1038/s41592-024-02197-7) +## Workspaces -- RΓΆst HL, Schmitt U, Aebersold R, MalmstrΓΆm L. pyOpenMS: a Python-based interface to the OpenMS mass-spectrometry algorithm library. Proteomics. 2014 Jan;14(1):74-7. [https://doi.org/10.1002/pmic.201300246](https://doi.org/10.1002/pmic.201300246). PMID: [24420968](https://pubmed.ncbi.nlm.nih.gov/24420968/). +Every analysis session runs in an isolated **workspace** that persists inputs, parameters, and results. In online deployments the workspace ID is part of the URL, so runs are resumable and shareable. +## Citation + +MΓΌller, T. D., Siraj, A., et al. *OpenMS WebApps: Building User-Friendly Solutions for MS Analysis.* Journal of Proteome Research (2025). [doi:10.1021/acs.jproteome.4c00872](https://doi.org/10.1021/acs.jproteome.4c00872) + +## References +- Pfeuffer, J., Bielow, C., Wein, S. et al. *OpenMS 3 enables reproducible analysis of large-scale mass spectrometry data.* Nat Methods 21, 365–367 (2024). [doi:10.1038/s41592-024-02197-7](https://doi.org/10.1038/s41592-024-02197-7) +- RΓΆst HL, Schmitt U, Aebersold R, MalmstrΓΆm L. *pyOpenMS: a Python-based interface to the OpenMS mass-spectrometry algorithm library.* Proteomics 14, 74–77 (2014). [doi:10.1002/pmic.201300246](https://doi.org/10.1002/pmic.201300246) diff --git a/default-parameters.json b/default-parameters.json index 7c084f8..11479a2 100644 --- a/default-parameters.json +++ b/default-parameters.json @@ -1,10 +1,9 @@ { - "example-workflow-selected-mzML-files": [], "image-format": "svg", - "2D-map-intensity-cutoff": 5000, - - "example-x-dimension": 10, - "example-y-dimension": 5, - - "controllo": false + "controllo": false, + "generate-library": false, + "library-use-fdr": false, + "library-psm-fdr": 0.01, + "library-generate-decoys": true, + "library-decoy-method": "shuffle" } diff --git a/pr-397.patch b/pr-397.patch deleted file mode 100644 index bb97819..0000000 --- a/pr-397.patch +++ /dev/null @@ -1,119 +0,0 @@ -From 268348f6da9809750e3116535f4f3fe9defc7ab1 Mon Sep 17 00:00:00 2001 -From: Yoo HoJun -Date: Wed, 15 Jul 2026 15:04:48 +0900 -Subject: [PATCH] Add support for boolean CLI flag parameters in TOPP tools - -Allows input_TOPP() to designate parameters as flags (present/absent) -instead of key-value pairs, persisted to params.json and session_state -so run_topp() can correctly build the command line. ---- - src/workflow/CommandExecutor.py | 53 +++++++++++++++++++++++---------- - src/workflow/StreamlitUI.py | 13 ++++++++ - 2 files changed, 51 insertions(+), 15 deletions(-) - -diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py -index 11bb1486..042c5e11 100644 ---- a/src/workflow/CommandExecutor.py -+++ b/src/workflow/CommandExecutor.py -@@ -268,6 +268,14 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool - - # Load merged parameters (_defaults + user overrides) for this tool instance - merged_params = self.parameter_manager.get_merged_params(params_key) -+ -+ # Load flag parameter names: params.json takes priority (survives session restart), -+ # session_state is the live fallback during the current session. -+ flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {}) -+ if not flag_map: -+ flag_map = st.session_state.get("_topp_flag_params", {}) -+ flag_params: set = set(flag_map.get(params_key, [])) -+ - # Construct commands for each process - for i in range(n_processes): - command = [tool] -@@ -288,25 +296,40 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool - command += [value[i]] - # Add merged TOPP tool parameters (_defaults + user overrides) - for k, v in merged_params.items(): -- command += [f"-{k}"] -- # Skip only empty strings (pass flag with no value) -- # Note: 0 and 0.0 are valid values, so use explicit check -- if v != "" and v is not None: -- if isinstance(v, str) and "\n" in v: -- command += v.split("\n") -+ if k in flag_params: -+ # CLI flag: include "-k" only when truthy, omit when false -+ if isinstance(v, str): -+ is_enabled = v.lower() == "true" - else: -- command += [str(v)] -+ is_enabled = bool(v) -+ if is_enabled: -+ command += [f"-{k}"] -+ continue -+ # Regular parameter: skip empty/None, append value otherwise -+ if v == "" or v is None: -+ continue -+ command += [f"-{k}"] -+ if isinstance(v, str) and "\n" in v: -+ command += v.split("\n") -+ else: -+ command += [str(v)] - # Add custom parameters - for k, v in custom_params.items(): -- command += [f"-{k}"] -- -- # Skip only empty strings (pass flag with no value) -- # Note: 0 and 0.0 are valid values, so use explicit check -- if v != "" and v is not None: -- if isinstance(v, list): -- command += [str(x) for x in v] -+ if k in flag_params: -+ if isinstance(v, str): -+ is_enabled = v.lower() == "true" - else: -- command += [str(v)] -+ is_enabled = bool(v) -+ if is_enabled: -+ command += [f"-{k}"] -+ continue -+ if v == "" or v is None: -+ continue -+ command += [f"-{k}"] -+ if isinstance(v, list): -+ command += [str(x) for x in v] -+ else: -+ command += [str(v)] - # Add threads parameter for TOPP tools - command += ["-threads", str(threads_per_command)] - commands.append(command) -diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py -index 9c24dc2c..4bd53b18 100644 ---- a/src/workflow/StreamlitUI.py -+++ b/src/workflow/StreamlitUI.py -@@ -826,6 +826,7 @@ def input_TOPP( - num_cols: int = 4, - exclude_parameters: List[str] = [], - include_parameters: List[str] = [], -+ flag_parameters: List[str] = [], - display_tool_name: bool = True, - display_subsections: bool = True, - display_subsection_tabs: bool = False, -@@ -862,6 +863,18 @@ def input_TOPP( - st.session_state["_topp_tool_instance_map"] = {} - st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name - -+ # Persist flag_parameters to session_state and params.json so run_topp -+ # can skip appending a value for these boolean CLI flags. -+ if "_topp_flag_params" not in st.session_state: -+ st.session_state["_topp_flag_params"] = {} -+ st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters) -+ _fp = self.parameter_manager.get_parameters_from_json() -+ if "_flag_params" not in _fp: -+ _fp["_flag_params"] = {} -+ _fp["_flag_params"][tool_instance_name] = list(flag_parameters) -+ with open(self.parameter_manager.params_file, "w", encoding="utf-8") as _f: -+ json.dump(_fp, _f, indent=4) -+ - if not display_subsections: - display_subsection_tabs = False - if display_subsection_tabs: diff --git a/presets.json b/presets.json index 1099493..d1878af 100644 --- a/presets.json +++ b/presets.json @@ -1,44 +1,19 @@ { "topp-workflow": { - "High Sensitivity": { - "_description": "Optimized for detecting low-abundance features with higher noise tolerance", - "FeatureFinderMetabo": { - "algorithm:common:noise_threshold_int": 500.0, - "algorithm:common:chrom_peak_snr": 2.0, - "algorithm:mtd:mass_error_ppm": 15.0 + "High Res MS": { + "_description": "Optimized for high-resolution MS2 (Orbitrap, TOF) when using ppm fragment tolerance", + "CometAdapter": { + "instrument": "high_res", + "fragment_mass_tolerance": 0.015, + "fragment_bin_offset": 0.0 } }, - "High Specificity": { - "_description": "Strict parameters for high-confidence feature detection", - "FeatureFinderMetabo": { - "algorithm:common:noise_threshold_int": 5000.0, - "algorithm:common:chrom_peak_snr": 5.0, - "algorithm:mtd:mass_error_ppm": 5.0 - } - }, - "Fast Analysis": { - "_description": "Faster processing with relaxed parameters for quick exploration", - "FeatureFinderMetabo": { - "algorithm:common:noise_threshold_int": 2000.0, - "algorithm:ffm:isotope_filtering_model": "none" - }, - "FeatureLinkerUnlabeledKD": { - "algorithm:link:rt_tol": 60.0, - "algorithm:link:mz_tol": 15.0 - } - }, - "Metabolomics Default": { - "_description": "Balanced parameters for general metabolomics analysis", - "FeatureFinderMetabo": { - "algorithm:common:noise_threshold_int": 1000.0, - "algorithm:common:chrom_peak_snr": 3.0, - "algorithm:mtd:mass_error_ppm": 10.0, - "algorithm:ffm:charge_lower_bound": 1, - "algorithm:ffm:charge_upper_bound": 3 - }, - "FeatureLinkerUnlabeledKD": { - "algorithm:link:rt_tol": 30.0, - "algorithm:link:mz_tol": 10.0 + "Low Res MS": { + "_description": "Optimized for low-resolution MS2 (Ion trap) when using ppm fragment tolerance", + "CometAdapter": { + "instrument": "low_res", + "fragment_mass_tolerance": 0.50025, + "fragment_bin_offset": 0.4 } } } diff --git a/requirements.txt b/requirements.txt index 0e26851..7caa20a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -140,7 +140,7 @@ xlsxwriter streamlit_plotly_events scipy scikit-learn -openms-insight>=0.1.13 +openms-insight==0.2.0 polars>=1.0.0 # Forces polars to prefer its most CPU-compatible native runtime. # Without this, polars can select an AVX-optimized runtime (e.g. runtime-32)