diff --git a/CHANGELOG.md b/CHANGELOG.md index fbea56d..600efdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ * Add new method: CellMapper (PR #31) +* Add new method: MCV-calibrated diffusion (`mcv_diffusion`) (PR #39) + ## MINOR CHANGES * Update the Viash version to 0.9.4 (PR #32) diff --git a/src/methods/mcv_diffusion/config.vsh.yaml b/src/methods/mcv_diffusion/config.vsh.yaml new file mode 100644 index 0000000..149f918 --- /dev/null +++ b/src/methods/mcv_diffusion/config.vsh.yaml @@ -0,0 +1,73 @@ +__merge__: ../../api/comp_method.yaml +name: mcv_diffusion +label: MCV-calibrated diffusion +summary: "kNN-graph diffusion in log-CP10k space whose diffusion time and restart weight are calibrated by molecular cross-validation on the training counts." +description: | + Cells are connected by a symmetrised k-nearest-neighbour graph built on the PCA of + log-normalised (log1p CP10k) highly variable genes. The log1p CP10k expression matrix Y is + smoothed by a lazy random walk with restart, Z = (1 - b) Y + b P^t Y, where P is the + row-normalised graph, and the denoised counts are expm1(Z) rescaled to each cell's library size. + Instead of fixing the amount of smoothing, the diffusion time t and the restart weight b are + chosen by molecular cross-validation (Batson et al., 2019) on the training data only: the + training molecules are binomially split into a train' and a test' part, every (t, b) on a + small grid is applied to train', and the pair with the lowest mean squared error between + log1p CP10k of the smoothed train' and of test' is used to denoise the full training matrix. + This lets the method adapt the smoothing strength to sequencing depth and technology + (UMI vs. full-length read counts) without access to the test molecules. +references: + doi: + - 10.1101/786269 +links: + documentation: https://github.com/openproblems-bio/task_denoising + repository: https://github.com/openproblems-bio/task_denoising +info: + preferred_normalization: counts + variants: + mcv_diffusion: +arguments: + - name: "--n_pcs" + type: integer + default: 50 + description: Number of principal components used to build the kNN graph. + - name: "--n_neighbors" + type: integer + default: 15 + description: Number of nearest neighbours (including the cell itself). + - name: "--t_grid" + type: integer + multiple: true + default: [1, 2, 3, 4, 6, 8, 12, 16] + description: Candidate diffusion times evaluated by molecular cross-validation. + - name: "--b_grid" + type: double + multiple: true + default: [0.3, 0.6, 0.8, 0.9, 1.0] + description: Candidate weights of the diffused profile (1 - b is the restart weight on the cell's own profile). + - name: "--mcv_frac" + type: double + default: 0.888888889 + description: Fraction of training molecules kept in train' for molecular cross-validation (8/9 gives a test' with the depth of the benchmark test set). + - name: "--mcv_gene_frac" + type: double + default: 0.25 + description: Fraction of genes on which the cross-validation error is evaluated. + - name: "--seed" + type: integer + default: 0 + description: Random seed for the molecule split, gene subset and PCA. +resources: + - type: python_script + path: script.py +engines: + - type: docker + image: openproblems/base_python:1 + setup: + - type: python + packages: + - scanpy>=1.10 + - scikit-learn +runners: + - type: executable + - type: nextflow + directives: + label: [midtime, highmem, midcpu] diff --git a/src/methods/mcv_diffusion/script.py b/src/methods/mcv_diffusion/script.py new file mode 100644 index 0000000..34aae10 --- /dev/null +++ b/src/methods/mcv_diffusion/script.py @@ -0,0 +1,141 @@ +import anndata as ad +import numpy as np +import scanpy as sc +import scipy.sparse as sp +from sklearn.neighbors import NearestNeighbors + +## VIASH START +par = { + "input_train": "resources_test/task_denoising/cxg_immune_cell_atlas/train.h5ad", + "output": "output_mcv_diffusion.h5ad", + "n_pcs": 50, + "n_neighbors": 15, + "t_grid": [1, 2, 3, 4, 6, 8, 12, 16], + "b_grid": [0.3, 0.6, 0.8, 0.9, 1.0], + "mcv_frac": 0.888888889, + "mcv_gene_frac": 0.25, + "seed": 0, +} +meta = {"name": "mcv_diffusion"} +## VIASH END + +# Denoising by kNN-graph diffusion in log-CP10k space, with the diffusion time t and the +# restart weight b calibrated by molecular cross-validation (MCV, Batson et al. 2019) on the +# training counts themselves: +# Y = log1p(1e4 * X / libsize) (per cell) +# Z = (1 - b) * Y + b * P^t Y (P: row-normalised symmetric kNN graph from PCA of Y) +# denoised = expm1(Z), rescaled per cell to the observed library size +# (t, b) minimise the MSE between log1p(CP10k) of the denoised train' split and of a held-out +# test' split obtained by binomially splitting the training molecules (no test data is used). + +rng = np.random.RandomState(par["seed"]) +GENE_CHUNK = 2048 + + +def log_cp10k(X): + lib = np.asarray(X.sum(1)).ravel().astype(np.float64) + lib[lib == 0] = 1 + Y = sp.csr_matrix(sp.diags(1e4 / lib) @ X, dtype=np.float32) + Y.data = np.log1p(Y.data) + return Y + + +def transition_matrix(X): + """Symmetrised, row-normalised kNN graph (incl. self) on the PCA of log-CP10k (scanpy HVG + PCA).""" + a = ad.AnnData(X=log_cp10k(X)) + sc.pp.highly_variable_genes(a) + n_pcs = min(par["n_pcs"], a.n_obs - 1, int(a.var["highly_variable"].sum()) - 1) + sc.pp.pca(a, n_comps=n_pcs, mask_var="highly_variable", random_state=par["seed"]) + E = a.obsm["X_pca"] + k = min(par["n_neighbors"], a.n_obs) + idx = NearestNeighbors(n_neighbors=k).fit(E).kneighbors(E, return_distance=False) + n = E.shape[0] + W = sp.csr_matrix((np.ones(idx.size, dtype=np.float32), (np.repeat(np.arange(n), k), idx.ravel())), shape=(n, n)) + W = (W + W.T) * 0.5 + return sp.csr_matrix(sp.diags(1.0 / np.asarray(W.sum(1)).ravel()) @ W, dtype=np.float32) + + +def diffusion_states(P, Yc, ts): + """Yield (t, P^t Yc) for t in ts (sorted), Yc dense.""" + Z = Yc + for t in range(1, max(ts) + 1): + Z = P @ Z + if t in ts: + yield t, Z + + +def calibrate(X): + """MCV: split molecules of X into train'/test', return (t, b) minimising the log-CP10k MSE on test'.""" + X = sp.csr_matrix(X) + tr = X.copy() + tr.data = rng.binomial(np.rint(X.data).astype(np.int32), par["mcv_frac"]).astype(np.float32) + te = X.copy() + te.data = (X.data - tr.data).astype(np.float32) + tr.eliminate_zeros(); te.eliminate_zeros() + keep = np.asarray(tr.sum(0)).ravel() > 0 + tr, te = tr[:, keep].tocsr(), te[:, keep].tocsc() + n, g = tr.shape + P = transition_matrix(tr) + Y = sp.csc_matrix(log_cp10k(tr)) + LT = sp.csc_matrix(log_cp10k(te)) + ts, bs = sorted(par["t_grid"]), list(par["b_grid"]) + # pass 1: per-cell totals of expm1(Z) for every (t, b) over all genes + rows = {} + for j in range(0, g, GENE_CHUNK): + B = Y[:, j:j + GENE_CHUNK].toarray() + for t, Zt in diffusion_states(P, B, ts): + for b in bs: + rows[(t, b)] = rows.get((t, b), 0) + np.expm1((1 - b) * B + b * Zt).sum(1, dtype=np.float64) + # pass 2: squared error on a random subset of genes (unbiased estimate of the MSE) + genes = np.sort(rng.choice(g, max(1, int(round(par["mcv_gene_frac"] * g))), replace=False)) + err = {key: 0.0 for key in rows} + for j in range(0, len(genes), GENE_CHUNK): + gj = genes[j:j + GENE_CHUNK] + B = Y[:, gj].toarray() + T = LT[:, gj].toarray() + for t, Zt in diffusion_states(P, B, ts): + for b in bs: + s = np.maximum(rows[(t, b)], 1e-30)[:, None] + V = np.log1p(np.expm1((1 - b) * B + b * Zt) / s * 1e4) + err[(t, b)] += float(((T - V) ** 2).sum()) + (t_best, b_best), e_best = min(err.items(), key=lambda kv: kv[1]) + print(f"MCV grid: best t={t_best} b={b_best} mse~{e_best / (n * len(genes)):.5f}", flush=True) + return t_best, b_best + + +def denoise(X, t, b): + X = sp.csr_matrix(X) + n, g = X.shape + P = transition_matrix(X) + Y = sp.csc_matrix(log_cp10k(X)) + out = np.empty((n, g), dtype=np.float32) + for j in range(0, g, GENE_CHUNK): + B = Y[:, j:j + GENE_CHUNK].toarray() + Zt = dict(diffusion_states(P, B, [t]))[t] + out[:, j:j + GENE_CHUNK] = np.expm1((1 - b) * B + b * Zt) + lib = np.asarray(X.sum(1)).ravel() + s = out.sum(1, dtype=np.float64) + s[s == 0] = 1 + out *= (lib / s).astype(np.float32)[:, None] + return out + + +print("Load data", flush=True) +input_train = ad.read_h5ad(par["input_train"]) +X = sp.csr_matrix(input_train.layers["counts"]).astype(np.float32) + +print("Calibrate diffusion time and restart weight by molecular cross-validation", flush=True) +t_best, b_best = calibrate(X) + +print("Denoise", flush=True) +denoised = denoise(X, t_best, b_best) + +output = ad.AnnData( + obs=input_train.obs[[]], + var=input_train.var[[]], + layers={"denoised": denoised}, + uns={"dataset_id": input_train.uns["dataset_id"], "method_id": meta["name"], + "mcv_t": t_best, "mcv_b": b_best}, +) +print("Write data", flush=True) +output.write_h5ad(par["output"], compression="gzip") diff --git a/src/workflows/run_benchmark/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml index 292007b..4698655 100644 --- a/src/workflows/run_benchmark/config.vsh.yaml +++ b/src/workflows/run_benchmark/config.vsh.yaml @@ -76,6 +76,7 @@ dependencies: - name: methods/dca - name: methods/knn_smoothing - name: methods/magic + - name: methods/mcv_diffusion - name: methods/scprint - name: metrics/mse - name: metrics/poisson diff --git a/src/workflows/run_benchmark/main.nf b/src/workflows/run_benchmark/main.nf index b3f45a0..4cb031b 100644 --- a/src/workflows/run_benchmark/main.nf +++ b/src/workflows/run_benchmark/main.nf @@ -22,6 +22,7 @@ workflow run_wf { dca, knn_smoothing, magic, + mcv_diffusion, scprint ]