Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

* Add new method: CellMapper (PR #31)

* Add new method: MCV-calibrated diffusion (`mcv_diffusion`) (PR #39)

## MINOR CHANGES

* Update the Viash version to 0.9.4 (PR #32)
Expand Down
73 changes: 73 additions & 0 deletions src/methods/mcv_diffusion/config.vsh.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
__merge__: ../../api/comp_method.yaml
name: mcv_diffusion
label: MCV-calibrated diffusion
summary: "kNN-graph diffusion in log-CP10k space whose diffusion time and restart weight are calibrated by molecular cross-validation on the training counts."
description: |
Cells are connected by a symmetrised k-nearest-neighbour graph built on the PCA of
log-normalised (log1p CP10k) highly variable genes. The log1p CP10k expression matrix Y is
smoothed by a lazy random walk with restart, Z = (1 - b) Y + b P^t Y, where P is the
row-normalised graph, and the denoised counts are expm1(Z) rescaled to each cell's library size.
Instead of fixing the amount of smoothing, the diffusion time t and the restart weight b are
chosen by molecular cross-validation (Batson et al., 2019) on the training data only: the
training molecules are binomially split into a train' and a test' part, every (t, b) on a
small grid is applied to train', and the pair with the lowest mean squared error between
log1p CP10k of the smoothed train' and of test' is used to denoise the full training matrix.
This lets the method adapt the smoothing strength to sequencing depth and technology
(UMI vs. full-length read counts) without access to the test molecules.
references:
doi:
- 10.1101/786269
links:
documentation: https://github.com/openproblems-bio/task_denoising
repository: https://github.com/openproblems-bio/task_denoising
info:
preferred_normalization: counts
variants:
mcv_diffusion:
arguments:
- name: "--n_pcs"
type: integer
default: 50
description: Number of principal components used to build the kNN graph.
- name: "--n_neighbors"
type: integer
default: 15
description: Number of nearest neighbours (including the cell itself).
- name: "--t_grid"
type: integer
multiple: true
default: [1, 2, 3, 4, 6, 8, 12, 16]
description: Candidate diffusion times evaluated by molecular cross-validation.
- name: "--b_grid"
type: double
multiple: true
default: [0.3, 0.6, 0.8, 0.9, 1.0]
description: Candidate weights of the diffused profile (1 - b is the restart weight on the cell's own profile).
- name: "--mcv_frac"
type: double
default: 0.888888889
description: Fraction of training molecules kept in train' for molecular cross-validation (8/9 gives a test' with the depth of the benchmark test set).
- name: "--mcv_gene_frac"
type: double
default: 0.25
description: Fraction of genes on which the cross-validation error is evaluated.
- name: "--seed"
type: integer
default: 0
description: Random seed for the molecule split, gene subset and PCA.
resources:
- type: python_script
path: script.py
engines:
- type: docker
image: openproblems/base_python:1
setup:
- type: python
packages:
- scanpy>=1.10
- scikit-learn
runners:
- type: executable
- type: nextflow
directives:
label: [midtime, highmem, midcpu]
141 changes: 141 additions & 0 deletions src/methods/mcv_diffusion/script.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
import anndata as ad
import numpy as np
import scanpy as sc
import scipy.sparse as sp
from sklearn.neighbors import NearestNeighbors

## VIASH START
par = {
"input_train": "resources_test/task_denoising/cxg_immune_cell_atlas/train.h5ad",
"output": "output_mcv_diffusion.h5ad",
"n_pcs": 50,
"n_neighbors": 15,
"t_grid": [1, 2, 3, 4, 6, 8, 12, 16],
"b_grid": [0.3, 0.6, 0.8, 0.9, 1.0],
"mcv_frac": 0.888888889,
"mcv_gene_frac": 0.25,
"seed": 0,
}
meta = {"name": "mcv_diffusion"}
## VIASH END

# Denoising by kNN-graph diffusion in log-CP10k space, with the diffusion time t and the
# restart weight b calibrated by molecular cross-validation (MCV, Batson et al. 2019) on the
# training counts themselves:
# Y = log1p(1e4 * X / libsize) (per cell)
# Z = (1 - b) * Y + b * P^t Y (P: row-normalised symmetric kNN graph from PCA of Y)
# denoised = expm1(Z), rescaled per cell to the observed library size
# (t, b) minimise the MSE between log1p(CP10k) of the denoised train' split and of a held-out
# test' split obtained by binomially splitting the training molecules (no test data is used).

rng = np.random.RandomState(par["seed"])
GENE_CHUNK = 2048


def log_cp10k(X):
lib = np.asarray(X.sum(1)).ravel().astype(np.float64)
lib[lib == 0] = 1
Y = sp.csr_matrix(sp.diags(1e4 / lib) @ X, dtype=np.float32)
Y.data = np.log1p(Y.data)
return Y


def transition_matrix(X):
"""Symmetrised, row-normalised kNN graph (incl. self) on the PCA of log-CP10k (scanpy HVG + PCA)."""
a = ad.AnnData(X=log_cp10k(X))
sc.pp.highly_variable_genes(a)
n_pcs = min(par["n_pcs"], a.n_obs - 1, int(a.var["highly_variable"].sum()) - 1)
sc.pp.pca(a, n_comps=n_pcs, mask_var="highly_variable", random_state=par["seed"])
E = a.obsm["X_pca"]
k = min(par["n_neighbors"], a.n_obs)
idx = NearestNeighbors(n_neighbors=k).fit(E).kneighbors(E, return_distance=False)
n = E.shape[0]
W = sp.csr_matrix((np.ones(idx.size, dtype=np.float32), (np.repeat(np.arange(n), k), idx.ravel())), shape=(n, n))
W = (W + W.T) * 0.5
return sp.csr_matrix(sp.diags(1.0 / np.asarray(W.sum(1)).ravel()) @ W, dtype=np.float32)


def diffusion_states(P, Yc, ts):
"""Yield (t, P^t Yc) for t in ts (sorted), Yc dense."""
Z = Yc
for t in range(1, max(ts) + 1):
Z = P @ Z
if t in ts:
yield t, Z


def calibrate(X):
"""MCV: split molecules of X into train'/test', return (t, b) minimising the log-CP10k MSE on test'."""
X = sp.csr_matrix(X)
tr = X.copy()
tr.data = rng.binomial(np.rint(X.data).astype(np.int32), par["mcv_frac"]).astype(np.float32)
te = X.copy()
te.data = (X.data - tr.data).astype(np.float32)
tr.eliminate_zeros(); te.eliminate_zeros()
keep = np.asarray(tr.sum(0)).ravel() > 0
tr, te = tr[:, keep].tocsr(), te[:, keep].tocsc()
n, g = tr.shape
P = transition_matrix(tr)
Y = sp.csc_matrix(log_cp10k(tr))
LT = sp.csc_matrix(log_cp10k(te))
ts, bs = sorted(par["t_grid"]), list(par["b_grid"])
# pass 1: per-cell totals of expm1(Z) for every (t, b) over all genes
rows = {}
for j in range(0, g, GENE_CHUNK):
B = Y[:, j:j + GENE_CHUNK].toarray()
for t, Zt in diffusion_states(P, B, ts):
for b in bs:
rows[(t, b)] = rows.get((t, b), 0) + np.expm1((1 - b) * B + b * Zt).sum(1, dtype=np.float64)
# pass 2: squared error on a random subset of genes (unbiased estimate of the MSE)
genes = np.sort(rng.choice(g, max(1, int(round(par["mcv_gene_frac"] * g))), replace=False))
err = {key: 0.0 for key in rows}
for j in range(0, len(genes), GENE_CHUNK):
gj = genes[j:j + GENE_CHUNK]
B = Y[:, gj].toarray()
T = LT[:, gj].toarray()
for t, Zt in diffusion_states(P, B, ts):
for b in bs:
s = np.maximum(rows[(t, b)], 1e-30)[:, None]
V = np.log1p(np.expm1((1 - b) * B + b * Zt) / s * 1e4)
err[(t, b)] += float(((T - V) ** 2).sum())
(t_best, b_best), e_best = min(err.items(), key=lambda kv: kv[1])
print(f"MCV grid: best t={t_best} b={b_best} mse~{e_best / (n * len(genes)):.5f}", flush=True)
return t_best, b_best


def denoise(X, t, b):
X = sp.csr_matrix(X)
n, g = X.shape
P = transition_matrix(X)
Y = sp.csc_matrix(log_cp10k(X))
out = np.empty((n, g), dtype=np.float32)
for j in range(0, g, GENE_CHUNK):
B = Y[:, j:j + GENE_CHUNK].toarray()
Zt = dict(diffusion_states(P, B, [t]))[t]
out[:, j:j + GENE_CHUNK] = np.expm1((1 - b) * B + b * Zt)
lib = np.asarray(X.sum(1)).ravel()
s = out.sum(1, dtype=np.float64)
s[s == 0] = 1
out *= (lib / s).astype(np.float32)[:, None]
return out


print("Load data", flush=True)
input_train = ad.read_h5ad(par["input_train"])
X = sp.csr_matrix(input_train.layers["counts"]).astype(np.float32)

print("Calibrate diffusion time and restart weight by molecular cross-validation", flush=True)
t_best, b_best = calibrate(X)

print("Denoise", flush=True)
denoised = denoise(X, t_best, b_best)

output = ad.AnnData(
obs=input_train.obs[[]],
var=input_train.var[[]],
layers={"denoised": denoised},
uns={"dataset_id": input_train.uns["dataset_id"], "method_id": meta["name"],
"mcv_t": t_best, "mcv_b": b_best},
)
print("Write data", flush=True)
output.write_h5ad(par["output"], compression="gzip")
1 change: 1 addition & 0 deletions src/workflows/run_benchmark/config.vsh.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ dependencies:
- name: methods/dca
- name: methods/knn_smoothing
- name: methods/magic
- name: methods/mcv_diffusion
- name: methods/scprint
- name: metrics/mse
- name: metrics/poisson
Expand Down
1 change: 1 addition & 0 deletions src/workflows/run_benchmark/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ workflow run_wf {
dca,
knn_smoothing,
magic,
mcv_diffusion,
scprint
]

Expand Down