Integrate scRNA-seq datasets

Show code cell content Hide code cell content

!lamin load test-scrna

💡 found cached instance metadata: /home/runner/.lamin/instance--testuser1--test-scrna.env

✅ loaded instance: testuser1/test-scrna

import lamindb as ln
import lnschema_bionty as lb
import pandas as pd
import anndata as ad

✅ loaded instance: testuser1/test-scrna (lamindb 0.50.3)

ln.track()

💡 notebook imports: anndata==0.9.2 lamindb==0.50.3 lnschema_bionty==0.29.3 pandas==1.5.3

🌱 saved: Transform(id='agayZTonayqAz8', name='Integrate scRNA-seq datasets', short_name='scrna2', stem_id='agayZTonayqA', version='0', type=notebook, updated_at=2023-08-12 05:42:48, created_by_id='DzTjkKse')

🌱 saved: Run(id='RNRplEpP0cewm9UCUjxh', run_at=2023-08-12 05:42:48, transform_id='agayZTonayqAz8', created_by_id='DzTjkKse')

Query files based on metadata

ln.File.filter(tissues__name__icontains="lymph node").distinct().df()

	storage_id	key	suffix	accessor	description	version	initial_version_id	size	hash	hash_type	transform_id	run_id	updated_at	created_by_id
id
WVRsT6NVgVneBJyB3wtW	RoLd8mpN	None	.h5ad	AnnData	Detmar22	None	None	17342743	rk5lSoJvz6PHRRjmcB919w	md5	Nv48yAceNSh8z8	n4BCYkDslFQ3C4DEggFa	2023-08-12 05:42:21	DzTjkKse
i2EH59qvxJdAePqFDolb	RoLd8mpN	None	.h5ad	AnnData	Conde22	None	None	28061905	3cIcmoqp1MxjX8NlRkKGlQ	md5	Nv48yAceNSh8z8	n4BCYkDslFQ3C4DEggFa	2023-08-12 05:42:37	DzTjkKse

ln.File.filter(cell_types__name__icontains="monocyte").distinct().df()

	storage_id	key	suffix	accessor	description	version	initial_version_id	size	hash	hash_type	transform_id	run_id	updated_at	created_by_id
id
i2EH59qvxJdAePqFDolb	RoLd8mpN	None	.h5ad	AnnData	Conde22	None	None	28061905	3cIcmoqp1MxjX8NlRkKGlQ	md5	Nv48yAceNSh8z8	n4BCYkDslFQ3C4DEggFa	2023-08-12 05:42:37	DzTjkKse
Qxsech1c8tvua5VTdiiE	RoLd8mpN	None	.h5ad	AnnData	10x reference pbmc68k	None	None	589484	eKVXV5okt5YRYjySMTKGEw	md5	Nv48yAceNSh8z8	n4BCYkDslFQ3C4DEggFa	2023-08-12 05:42:42	DzTjkKse

ln.File.filter(labels__name="female").distinct().df()

	storage_id	key	suffix	accessor	description	version	initial_version_id	size	hash	hash_type	transform_id	run_id	updated_at	created_by_id
id
WVRsT6NVgVneBJyB3wtW	RoLd8mpN	None	.h5ad	AnnData	Detmar22	None	None	17342743	rk5lSoJvz6PHRRjmcB919w	md5	Nv48yAceNSh8z8	n4BCYkDslFQ3C4DEggFa	2023-08-12 05:42:21	DzTjkKse

Intersect measured genes between two datasets

file1 = ln.File.filter(description="Conde22").one()
file2 = ln.File.filter(description="10x reference pbmc68k").one()

file1.describe()

💡 File(id=i2EH59qvxJdAePqFDolb, key=None, suffix=.h5ad, accessor=AnnData, description=Conde22, version=None, size=28061905, hash=3cIcmoqp1MxjX8NlRkKGlQ, hash_type=md5, created_at=2023-08-12 05:42:37.808600+00:00, updated_at=2023-08-12 05:42:37.808621+00:00)

Provenance:
    🗃️ storage: Storage(id='RoLd8mpN', root='/home/runner/work/lamin-usecases/lamin-usecases/docs/test-scrna', type='local', updated_at=2023-08-12 05:42:46, created_by_id='DzTjkKse')
    📎 initial_version: None
    📔 transform: Transform(id='Nv48yAceNSh8z8', name='Curate & link scRNA-seq datasets', short_name='scrna', stem_id='Nv48yAceNSh8', version='0', type='notebook', updated_at=2023-08-12 05:42:41, created_by_id='DzTjkKse')
    🚗 run: Run(id='n4BCYkDslFQ3C4DEggFa', run_at=2023-08-12 05:42:09, transform_id='Nv48yAceNSh8z8', created_by_id='DzTjkKse')
    👤 created_by: User(id='DzTjkKse', handle='testuser1', email='testuser1@lamin.ai', name='Test User1', updated_at=2023-08-12 05:42:46)
Features:
  🗺️ var (X):
    🔗 index (36503, bionty.Gene.id): ['2q1fHErbQUpM', 'PHZBB35I9uRP', '78bENBCA4g4a', 'wrrLPKIosD34', 'lw4v5rHEri8X'...]
  🗺️ external:
    🔗 species (1, bionty.Species): ['human']
  🗺️ obs (metadata):
    🔗 cell_type (32, bionty.CellType): ['CD4-positive helper T cell', 'macrophage', 'gamma-delta T cell', 'mucosal invariant T cell', 'alveolar macrophage']
    🔗 assay (3, bionty.ExperimentalFactor): ["10x 5' v1", "10x 5' v2", "10x 3' v3"]
    🔗 tissue (17, bionty.Tissue): ['mesenteric lymph node', 'lung', 'duodenum', 'omentum', 'bone marrow']
    🔗 donor (12, core.Label): ['640C', 'A36', 'A35', 'D496', '621B']

file1.view_lineage()

file2.describe()

💡 File(id=Qxsech1c8tvua5VTdiiE, key=None, suffix=.h5ad, accessor=AnnData, description=10x reference pbmc68k, version=None, size=589484, hash=eKVXV5okt5YRYjySMTKGEw, hash_type=md5, created_at=2023-08-12 05:42:42.147677+00:00, updated_at=2023-08-12 05:42:42.147702+00:00)

Provenance:
    🗃️ storage: Storage(id='RoLd8mpN', root='/home/runner/work/lamin-usecases/lamin-usecases/docs/test-scrna', type='local', updated_at=2023-08-12 05:42:46, created_by_id='DzTjkKse')
    📎 initial_version: None
    📔 transform: Transform(id='Nv48yAceNSh8z8', name='Curate & link scRNA-seq datasets', short_name='scrna', stem_id='Nv48yAceNSh8', version='0', type='notebook', updated_at=2023-08-12 05:42:41, created_by_id='DzTjkKse')
    🚗 run: Run(id='n4BCYkDslFQ3C4DEggFa', run_at=2023-08-12 05:42:09, transform_id='Nv48yAceNSh8z8', created_by_id='DzTjkKse')
    👤 created_by: User(id='DzTjkKse', handle='testuser1', email='testuser1@lamin.ai', name='Test User1', updated_at=2023-08-12 05:42:46)
Features:
  🗺️ var (X):
    🔗 index (695, bionty.Gene.id): ['kJ2XCUiYTklp', 'K3Tp6gWgYH1m', '3u6gS0Aq7a2c', 'FXX0c1917vs2', 'hrxj5Sa2bEqk'...]
  🗺️ obs (metadata):
    🔗 cell_type (9, bionty.CellType): ['B cell, CD19-positive', 'effector memory CD4-positive, alpha-beta T cell, terminally differentiated', 'conventional dendritic cell', 'cytotoxic T cell', 'dendritic cell']

file2.view_lineage()

file1_adata = file1.load()
file2_adata = file2.load()

💡 adding file i2EH59qvxJdAePqFDolb as input for run RNRplEpP0cewm9UCUjxh, adding parent transform Nv48yAceNSh8z8

💡 adding file Qxsech1c8tvua5VTdiiE as input for run RNRplEpP0cewm9UCUjxh, adding parent transform Nv48yAceNSh8z8

file2_adata.obs.cell_type.head()

index
GCAGGGCTGGATTC-1                                       dendritic cell
CTTTAGTGGTTACG-6                                B cell, CD19-positive
TGACTGGAACCATG-7                                       dendritic cell
TCAATCACCCTTCG-8                                B cell, CD19-positive
CGTTATACAGTACC-8    effector memory CD4-positive, alpha-beta T cel...
Name: cell_type, dtype: category
Categories (9, object): ['CD8-positive, CD25-positive, alpha-beta regul..., 'effector memory CD4-positive, alpha-beta T ce..., 'cytotoxic T cell', 'CD38-negative naive B cell', ..., 'B cell, CD19-positive', 'conventional dendritic cell', 'CD16-positive, CD56-dim natural killer cell, ..., 'dendritic cell']

Here we compute shared genes without loading files:

file1_genes = file1.features["var"]
file2_genes = file2.features["var"]

shared_genes = file1_genes & file2_genes
shared_genes.list("symbol")[:10]

['CTSH',
 'GNG7',
 'ZFP36L2',
 'BANK1',
 'KLF6',
 'MAD2L2',
 'CCDC115',
 'TBC1D10C',
 'FCER1A',
 'CWC15']

We also need to convert the ensembl_gene_id to symbol for file2 so that they can be concatenated:

mapper = (
    pd.DataFrame(file2_genes.values_list("ensembl_gene_id", "symbol"))
    .drop_duplicates(0)
    .set_index(0)[1]
)
mapper.head()

0
ENSG00000204482          LST1
ENSG00000124766          SOX4
ENSG00000158869        FCER1G
ENSG00000100100       PIK3IP1
ENSG00000070526    ST6GALNAC1
Name: 1, dtype: object

file1_adata.var.rename(index=mapper, inplace=True)

Intersect cell types

file1_celltypes = file1.cell_types.all()
file2_celltypes = file2.cell_types.all()

shared_celltypes = file1_celltypes & file2_celltypes
shared_celltypes_names = shared_celltypes.list("name")
shared_celltypes_names

['CD16-positive, CD56-dim natural killer cell, human',
 'conventional dendritic cell']

We can now subset the two datasets by shared cell types:

file1_adata_subset = file1_adata[
    file1_adata.obs["cell_type"].isin(shared_celltypes_names)
]
file1_adata_subset.obs["cell_type"].value_counts()

CD16-positive, CD56-dim natural killer cell, human    114
conventional dendritic cell                             7
Name: cell_type, dtype: int64

file2_adata_subset = file2_adata[
    file2_adata.obs["cell_type"].isin(shared_celltypes_names)
]
file2_adata_subset.obs["cell_type"].value_counts()

CD16-positive, CD56-dim natural killer cell, human    3
conventional dendritic cell                           2
Name: cell_type, dtype: int64

adata_concat = ad.concat(
    [file1_adata_subset, file2_adata_subset],
    label="file",
    keys=[file1.description, file2.description],
)
adata_concat

AnnData object with n_obs × n_vars = 126 × 695
    obs: 'cell_type', 'file'
    obsm: 'X_umap'

adata_concat.obs.value_counts()

cell_type                                           file                 
CD16-positive, CD56-dim natural killer cell, human  Conde22                  114
conventional dendritic cell                         Conde22                    7
CD16-positive, CD56-dim natural killer cell, human  10x reference pbmc68k      3
conventional dendritic cell                         10x reference pbmc68k      2
dtype: int64

Show code cell content Hide code cell content

!lamin delete test-scrna
!rm -r ./test-scrna

💡 deleting instance testuser1/test-scrna
✅     deleted instance settings file: /home/runner/.lamin/instance--testuser1--test-scrna.env

✅     instance cache deleted
✅     deleted '.lndb' sqlite file
🔶     consider manually delete your stored data: /home/runner/work/lamin-usecases/lamin-usecases/docs/test-scrna