Combine open OpenDDA PRs and guide biologists from upload to analysis

.gitattributes

@@ -1,3 +1,9 @@
+# Shell scripts are executed by /bin/sh inside Linux containers and by POSIX
+# shells in tests/test_seed_demos.py. A CRLF checkout on Windows breaks both:
+# sh reports "not found" on every carriage return, and a shebang with a
+# trailing CR fails to exec. Pin them to LF regardless of core.autocrlf.
+*.sh text eol=lf
+
 # MS data files carry byte offsets (indexed mzML); never convert line endings.
 *.mzML -text
 *.fasta -text

Dockerfile

@@ -18,6 +18,10 @@ ARG GITHUB_REPO=quantms-web
 
 USER root
 
+# Keep apt from stopping at interactive prompts: tzdata, pulled in by
+# later packages, otherwise waits for a timezone and the build hangs.
+ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC
+
 # Install required Ubuntu packages.
 RUN apt-get -y update
 RUN apt-get install -y --no-install-recommends --no-install-suggests g++ autoconf automake patch libtool make git gpg wget ca-certificates curl jq libgtk2.0-dev openjdk-8-jdk cron
@@ -132,6 +136,12 @@ FROM compile-openms AS run-app
 RUN apt-get update && apt-get install -y --no-install-recommends redis-server nginx \
     && rm -rf /var/lib/apt/lists/*
 
+# Mono runs the ThermoRawFileParser (THIRDPARTY/All, a .NET Framework build)
+# that FileConverter calls to convert Thermo RAW files to mzML on Linux.
+RUN apt-get update && apt-get install -y --no-install-recommends mono-complete \
+    && rm -rf /var/lib/apt/lists/* \
+    && mono --version
+
 # Create Redis data directory. Default 0755 root-owned is enough: the docker
 # entrypoint runs as root (can write regardless of mode), and the apptainer
 # entrypoint relocates Redis state to /tmp/openms-runtime-* so this dir is

Dockerfile.arm

@@ -18,6 +18,10 @@ ARG GITHUB_REPO=quantms-web
 
 USER root
 
+# Keep apt from stopping at interactive prompts: tzdata, pulled in by
+# later packages, otherwise waits for a timezone and the build hangs.
+ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC
+
 # Install required Ubuntu packages.
 RUN apt-get -y update
 RUN apt-get install -y --no-install-recommends --no-install-suggests g++ autoconf automake patch libtool make git gpg wget ca-certificates curl jq libgtk2.0-dev openjdk-8-jdk cron cmake
@@ -160,6 +164,12 @@ FROM compile-openms AS run-app
 RUN apt-get update && apt-get install -y --no-install-recommends redis-server nginx \
     && rm -rf /var/lib/apt/lists/*
 
+# Mono runs the ThermoRawFileParser (THIRDPARTY/All, a .NET Framework build)
+# that FileConverter calls to convert Thermo RAW files to mzML on Linux.
+RUN apt-get update && apt-get install -y --no-install-recommends mono-complete \
+    && rm -rf /var/lib/apt/lists/* \
+    && mono --version
+
 # Create Redis data directory. Default 0755 root-owned is enough: the docker
 # entrypoint runs as root (can write regardless of mode), and the apptainer
 # entrypoint relocates Redis state to /tmp/openms-runtime-* so this dir is

Dockerfile_simple

@@ -20,6 +20,10 @@ ARG GITHUB_REPO=quantms-web
 # Step 1: set up a sane build system
 USER root
 
+# Keep apt from stopping at interactive prompts: tzdata, pulled in by
+# later packages, otherwise waits for a timezone and the build hangs.
+ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC
+
 RUN apt-get -y update
 # note: streamlit in docker needs libgtk2.0-dev (see https://yugdamor.medium.com/importerror-libgthread-2-0-so-0-cannot-open-shared-object-file-no-such-file-or-directory-895b94a7827b)
 RUN apt-get install -y --no-install-recommends --no-install-suggests wget ca-certificates libgtk2.0-dev curl jq cron nginx

Dockerfile_simple.arm

@@ -20,6 +20,10 @@ ARG GITHUB_REPO=quantms-web
 # Step 1: set up a sane build system
 USER root
 
+# Keep apt from stopping at interactive prompts: tzdata, pulled in by
+# later packages, otherwise waits for a timezone and the build hangs.
+ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC
+
 RUN apt-get -y update
 # note: streamlit in docker needs libgtk2.0-dev (see https://yugdamor.medium.com/importerror-libgthread-2-0-so-0-cannot-open-shared-object-file-no-such-file-or-directory-895b94a7827b)
 # build-essential is needed on arm64: pyprophet (a quantms-web requirement, not a

app.py

@@ -16,37 +16,50 @@
             st.session_state.settings = json.load(f)
 
 if __name__ == '__main__':
+    # Sections follow the path a user takes: set up and run, read the main
+    # answers, dig deeper. Expert and quality-control pages come last.
     pages = {
-        "Welcome": [
-            st.Page(Path("content", "quickstart.py"), title="Quickstart", icon="👋"),
+        "Start here": [
+            st.Page(Path("content", "quickstart.py"), title="Home", icon="🏠", default=True),
         ],
-        "Workflow": [
-            st.Page(Path("content", "workflow_fileupload.py"), title="File Upload", icon="📁"),
-            st.Page(Path("content", "workflow_configure.py"), title="Configure", icon="⚙️"),
-            st.Page(Path("content", "workflow_run.py"), title="Run", icon="🚀"),
+        "Step 1 · Set up and run": [
+            st.Page(Path("content", "workflow_fileupload.py"), title="Upload data", icon="📁"),
+            st.Page(Path("content", "workflow_configure.py"), title="Describe experiment", icon="⚙️"),
+            st.Page(Path("content", "workflow_run.py"), title="Run analysis", icon="🚀"),
         ],
-        "Workflow Results": [
-            st.Page(Path("content", "results_database_search.py"), title="Database Search", icon="🔬"),
-            st.Page(Path("content", "results_rescoring.py"), title="Rescoring", icon="📈"),
-            st.Page(Path("content", "results_filtered.py"), title="Filtered PSMs", icon="🎯"),
-            st.Page(Path("content", "results_abundance.py"), title="Protein Abundance", icon="📋"),
-            st.Page(Path("content", "results_download.py"), title="Download", icon="⬇️"),
+        "Step 2 · Main results": [
+            st.Page(Path("content", "results_overview.py"), title="Run summary", icon="🧾"),
+            st.Page(Path("content", "compare_groups.py"), title="Compare groups", icon="⚖️"),
+            st.Page(Path("content", "results_volcano.py"), title="Volcano plot", icon="🌋"),
+            st.Page(Path("content", "results_proteins.py"), title="Identified proteins", icon="🧩"),
         ],
-        # Steps run in order; each one works on the previous step's output.
-        "Downstream Analysis": [
-            st.Page(Path("content", "filtering.py"), title="1. Filtering", icon="🧹"),
-            st.Page(Path("content", "imputation.py"), title="2. Imputation", icon="🩹"),
-            st.Page(Path("content", "normalization.py"), title="3. Normalization", icon="⚖️"),
-            st.Page(Path("content", "statistical.py"), title="4. Statistics", icon="🔢"),
-        ],
-        # Every plot reads the latest Downstream Analysis output.
-        "Downstream Plots": [
-            st.Page(Path("content", "results_volcano.py"), title="Volcano", icon="🌋"),
-            st.Page(Path("content", "results_pca.py"), title="PCA", icon="📊"),
+        # The sidebar shows the first ten pages and folds the rest behind
+        # "View more", so everything from Clustered heatmap on is one click away.
+        "Step 3 · Explore further": [
+            st.Page(Path("content", "results_pca.py"), title="Sample similarity (PCA)", icon="📊"),
+            st.Page(Path("content", "enrichment.py"), title="GO enrichment", icon="📉"),
+            st.Page(Path("content", "results_heatmap_clustered.py"), title="Clustered heatmap", icon="🧬"),
+            st.Page(Path("content", "results_abundance.py"), title="Protein abundances", icon="📋"),
+            st.Page(Path("content", "results_samples.py"), title="Per-sample counts", icon="🧪"),
             st.Page(Path("content", "results_heatmap.py"), title="Heatmap", icon="🔥"),
-            st.Page(Path("content", "results_heatmap_clustered.py"), title="Clustered Heatmap", icon="🧬"),
-            st.Page(Path("content", "enrichment.py"), title="Pathway Analysis", icon="📉"),
-        ]
+        ],
+        "Export": [
+            st.Page(Path("content", "results_download.py"), title="Download results", icon="⬇️"),
+        ],
+        # The four steps Compare groups runs with recommended settings, one at a time.
+        "Fine-tune the comparison": [
+            st.Page(Path("content", "filtering.py"), title="1. Filter proteins", icon="🧹"),
+            st.Page(Path("content", "imputation.py"), title="2. Missing values", icon="🩹"),
+            st.Page(Path("content", "normalization.py"), title="3. Normalize", icon="⚖️"),
+            st.Page(Path("content", "statistical.py"), title="4. Statistical test", icon="🔢"),
+        ],
+        "Quality control and details": [
+            st.Page(Path("content", "results_qc.py"), title="Identification quality", icon="🩺"),
+            st.Page(Path("content", "results_filtered.py"), title="Peptide-spectrum matches", icon="🎯"),
+            st.Page(Path("content", "results_mass_shifts.py"), title="Mass shifts", icon="📐"),
+            st.Page(Path("content", "results_rescoring.py"), title="Rescored hits (Percolator)", icon="📈"),
+            st.Page(Path("content", "results_database_search.py"), title="Raw search hits (Comet)", icon="🔬"),
+        ],
     }
 
     pg = st.navigation(pages)

content/compare_groups.py

@@ -0,0 +1,139 @@
+"""Compare Groups Page: which proteins change between the experimental groups."""
+import json
+
+import streamlit as st
+
+from src.common.common import page_setup, save_params
+from src.common.differential import STEPS, comparison_groups, count_significant
+from src.common.guide import (
+    ensure_group_comparison,
+    needs_results,
+    next_steps,
+    page_intro,
+    page_link,
+    show_comparison_settings,
+)
+from src.common.insight_tables import show_insight_table
+from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
+
+params = page_setup()
+st.title("Compare groups")
+
+page_intro(
+    "Which proteins are more or less abundant in one group of samples than in the other? "
+    "The comparison runs automatically with settings that suit most experiments, label-free and TMT alike.",
+    background="""
+The comparison runs four steps on the protein abundance table:
+
+1. **Filter proteins.** Proteins seen in only a few samples cannot be compared
+   reliably, so a protein is kept only if it was measured in at least half of the
+   samples of at least one group.
+2. **Fill in missing values.** A protein that is missing in a sample was usually
+   too faint to detect there. It gets that protein's lowest measured value, so a
+   protein switched off in one group still shows up as down-regulated.
+3. **Normalize.** Intensities are log2-transformed (so a difference of 1 means
+   twice as much) and each sample is shifted so its median matches the others,
+   which corrects for unequal amounts of material loaded.
+4. **Test.** A moderated t-test (similar to *limma*) compares the groups protein
+   by protein. Because thousands of proteins are tested, the p-values are
+   adjusted with the Benjamini-Hochberg method: an **adjusted p-value of 0.05**
+   means about 5 % of the proteins called significant are expected to be false
+   positives.
+
+**log2 fold change** is the difference of the group means on the log2 scale:
++1 means twice as abundant, −1 half as abundant.
+""",
+    background_title="What happens here, in plain words",
+)
+
+if "workspace" not in st.session_state:
+    st.warning("Please initialize your workspace first.")
+    st.stop()
+
+result = get_abundance_data(st.session_state["workspace"])
+if result is None:
+    needs_results()
+
+pivot_df, _, group_map = result
+id_col = get_id_column(st.session_state["workspace"], pivot_df)
+sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map)
+sample_cols = [c for c in pivot_df.columns if c in sample_group_map]
+
+st.subheader("Your groups")
+groups: dict[str, list[str]] = {}
+for s in sample_cols:
+    groups.setdefault(sample_group_map[s], []).append(s)
+cols = st.columns(max(len(groups), 1))
+for col, (group, members) in zip(cols, sorted(groups.items())):
+    col.metric(group, f"{len(members)} sample{'' if len(members) == 1 else 's'}")
+if "Unassigned" in groups:
+    st.warning(
+        "Some samples have no group, so they are left out of the comparison. Give each a group on "
+        "the **Describe experiment** page (for TMT, type **skip** for channels you do not want "
+        "compared). The comparison updates without running the analysis again."
+    )
+    page_link("configure")
+
+if st.button(
+    "Start over with the recommended settings",
+    help="Discard steps changed on the Fine-tune pages and compare the groups again with the defaults.",
+):
+    for key in [*STEPS, "normalized_transform", "comparison_settings"]:
+        st.session_state.pop(key, None)
+    with open("default-parameters.json", "r", encoding="utf-8") as f:
+        defaults = json.load(f)
+    for key, value in defaults.items():
+        if key.startswith("postproc-"):
+            params[key] = value
+            st.session_state.pop(key, None)
+    save_params(params)
+
+statistics_df = ensure_group_comparison(params)
+if statistics_df is None:
+    st.stop()
+
+direction = comparison_groups(sample_group_map, sample_cols)
+up, down = count_significant(statistics_df)
+st.subheader("Result")
+m = st.columns(3)
+m[0].metric("Proteins compared", f"{statistics_df['p-adj'].notna().sum():,}")
+if direction:
+    ref, comp = direction
+    m[1].metric(f"Higher in {comp}", f"{up:,}", help="Adjusted p-value ≤ 0.05 and at least twice as abundant.")
+    m[2].metric(f"Lower in {comp}", f"{down:,}", help="Adjusted p-value ≤ 0.05 and at most half as abundant.")
+    note = f"A positive log2 fold change means higher in **{comp}** than in **{ref}**."
+    if len(groups) > 2:
+        note += (
+            " With more than two groups the p-value tests whether any group differs; the fold "
+            f"change still compares **{comp}** with **{ref}**."
+        )
+    st.caption(note + " Counts use |log2 fold change| ≥ 1 and adjusted p-value ≤ 0.05; explore other cut-offs on the volcano plot.")
+show_comparison_settings()
+
+st.subheader("All compared proteins, most significant first")
+table = statistics_df.sort_values("p-adj", na_position="last")[[id_col, "log2FC", "p-adj", "p-value"]].copy()
+if direction:
+    sig = (table["p-adj"] <= 0.05) & (table["log2FC"].abs() >= 1)
+    table.insert(1, "Change", "–")
+    table.loc[sig & (table["log2FC"] > 0), "Change"] = f"higher in {direction[1]}"
+    table.loc[sig & (table["log2FC"] < 0), "Change"] = f"lower in {direction[1]}"
+for col in ("p-adj", "p-value"):
+    # As text with three significant digits: the table stores floats as float32,
+    # which would show 0.155 as 0.1550000011920929. Rows are already in p-value order.
+    table[col] = table[col].map(lambda v: "" if v != v else f"{v:.3g}")
+table = table.rename(columns={
+    id_col: "Protein", "log2FC": "log2 fold change", "p-adj": "adjusted p-value",
+})
+show_insight_table(
+    table, name="compare-groups", layout="fitColumns",
+    column_overrides={
+        "log2 fold change": {"formatter": "money", "formatterParams": {"thousand": False, "precision": 2}},
+    },
+)
+st.caption("The table's download button saves it as CSV, in the current sort and filter.")
+
+next_steps(
+    ("volcano", "See every protein's fold change against its significance and pick cut-offs."),
+    ("pca", "Check that samples of the same group look alike."),
+    ("enrichment", "Find biological processes over-represented among the changed proteins."),
+)