From 96972ebd3d418e10c76ca2a69c6ea213b60101f1 Mon Sep 17 00:00:00 2001 From: Marina Vabistsevits Date: Mon, 29 Nov 2021 16:02:44 +0000 Subject: [PATCH 1/4] fix issue with mapping all drugs to diseases --- .../rels/opentargets_drug_disease.py | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/workflow/scripts/processing/rels/opentargets_drug_disease.py b/workflow/scripts/processing/rels/opentargets_drug_disease.py index cbbd728..81b6c93 100644 --- a/workflow/scripts/processing/rels/opentargets_drug_disease.py +++ b/workflow/scripts/processing/rels/opentargets_drug_disease.py @@ -22,15 +22,25 @@ FILE = get_source(meta_id,1) -def get_disease_data(): +def get_disease_data(): + # query the graph for all mondo_ids and efo_ids separately and then merge them on mondo_id col + # we have to do two separate queries because efo column has to be unwind, and + # that command retains only rows where the operation is performed, so we lose all mondo_ids that have nan in efo col + driver = neo4j_connect() session = driver.session() - query = """ - match (d:Disease) unwind(d.efo) as mondo_efo_id return d.id as disease_id, mondo_efo_id; - """ - query_data = session.run(query).data() - df = pd.json_normalize(query_data) - logger.info(df) + + query1 = """ match (d:Disease) return d.id as mondo_id """ + query_data1 = session.run(query1).data() + mondo_only = pd.json_normalize(query_data1) + + query2 = """match (d:Disease) unwind(d.efo) as efo_id return d.id as mondo_id , efo_id""" + query_data2 = session.run(query2).data() + mondo_w_efo = pd.json_normalize(query_data2) + mondo_w_efo['efo_id'] = 'http://www.ebi.ac.uk/efo/EFO_' + mondo_w_efo['efo_id'].astype(str) + + df = pd.merge(mondo_only, mondo_w_efo, how='outer', on='mondo_id') + df = df.drop_duplicates() return df @@ -38,27 +48,27 @@ def process(): data = os.path.join(dataDir, FILE) # not sure why double quotes weren't being handled properly, added engine param df = pd.read_csv(data, sep=",", engine="python") + df = df.rename(columns={"efo_id": "disease_id"}) logger.info(df.shape) logger.info("\n {}", df.head()) + df = df[["molecule_name", "disease_id"]] - #get disease data + # get disease data from the graph disease_df = get_disease_data() - disease_df['mondo_efo_id'] = 'http://www.ebi.ac.uk/efo/EFO_'+disease_df['mondo_efo_id'].astype(str) logger.info(disease_df) - keep_cols = [ - "molecule_name", - "efo_id", - ] - df = df[keep_cols] - mondo_match = pd.merge(df,disease_df,left_on='efo_id',right_on='disease_id')[['molecule_name','disease_id']] + # join df (OT data) and disease_df (graph) on mondo_id + mondo_match = pd.merge(df, disease_df, left_on='disease_id', right_on='mondo_id')[['molecule_name', 'disease_id']] + mondo_match.drop_duplicates(inplace=True) #logger.info(mondo_match) - efo_match = pd.merge(df,disease_df,left_on='efo_id',right_on='mondo_efo_id')[['molecule_name','disease_id']] - #logger.info(efo_match) + # join on efo_id, but keep the corresponding mondo_id, as this is what used for mapping + efo_match = pd.merge(df, disease_df, left_on='disease_id', right_on='efo_id')[['molecule_name', 'mondo_id']] + efo_match = efo_match.rename(columns={"mondo_id": "disease_id"}) + efo_match.drop_duplicates(inplace=True) - cat_df = pd.concat([mondo_match,efo_match]) + cat_df = pd.concat([mondo_match, efo_match]) logger.info(cat_df.shape) cat_df.drop_duplicates(inplace=True) From 74f137b7fa3cac18df1eaf246509ba74521b3c28 Mon Sep 17 00:00:00 2001 From: Marina Vabistsevits Date: Thu, 2 Dec 2021 15:35:09 +0000 Subject: [PATCH 2/4] drop extra column in new arxiv data --- workflow/scripts/processing/nodes/literature/semrep-biorxiv.py | 1 + workflow/scripts/processing/nodes/literature/semrep-medrxiv.py | 1 + 2 files changed, 2 insertions(+) diff --git a/workflow/scripts/processing/nodes/literature/semrep-biorxiv.py b/workflow/scripts/processing/nodes/literature/semrep-biorxiv.py index 7494804..b0368f0 100644 --- a/workflow/scripts/processing/nodes/literature/semrep-biorxiv.py +++ b/workflow/scripts/processing/nodes/literature/semrep-biorxiv.py @@ -63,6 +63,7 @@ def merge_data(lit_data, sem_data): "license", "category", "abstract", + 'jatsxml', ], axis=1, inplace=True, diff --git a/workflow/scripts/processing/nodes/literature/semrep-medrxiv.py b/workflow/scripts/processing/nodes/literature/semrep-medrxiv.py index 511c0d5..0ec9cd7 100644 --- a/workflow/scripts/processing/nodes/literature/semrep-medrxiv.py +++ b/workflow/scripts/processing/nodes/literature/semrep-medrxiv.py @@ -63,6 +63,7 @@ def merge_data(lit_data, sem_data): "license", "category", "abstract", + 'jatsxml', ], axis=1, inplace=True, From a0a250fefdf2e7bfaf69ec9240ee49507dccc411 Mon Sep 17 00:00:00 2001 From: Marina Vabistsevits Date: Thu, 2 Dec 2021 15:37:18 +0000 Subject: [PATCH 3/4] minor fixes --- .../processing/rels/opentargets_drug_disease.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/workflow/scripts/processing/rels/opentargets_drug_disease.py b/workflow/scripts/processing/rels/opentargets_drug_disease.py index 81b6c93..55158b1 100644 --- a/workflow/scripts/processing/rels/opentargets_drug_disease.py +++ b/workflow/scripts/processing/rels/opentargets_drug_disease.py @@ -39,9 +39,9 @@ def get_disease_data(): mondo_w_efo = pd.json_normalize(query_data2) mondo_w_efo['efo_id'] = 'http://www.ebi.ac.uk/efo/EFO_' + mondo_w_efo['efo_id'].astype(str) - df = pd.merge(mondo_only, mondo_w_efo, how='outer', on='mondo_id') - df = df.drop_duplicates() - return df + disease_df = pd.merge(mondo_only, mondo_w_efo, how='outer', on='mondo_id') + disease_df = disease_df.drop_duplicates() + return disease_df def process(): @@ -57,16 +57,16 @@ def process(): disease_df = get_disease_data() logger.info(disease_df) - # join df (OT data) and disease_df (graph) on mondo_id mondo_match = pd.merge(df, disease_df, left_on='disease_id', right_on='mondo_id')[['molecule_name', 'disease_id']] mondo_match.drop_duplicates(inplace=True) - #logger.info(mondo_match) + # logger.info(mondo_match) # join on efo_id, but keep the corresponding mondo_id, as this is what used for mapping efo_match = pd.merge(df, disease_df, left_on='disease_id', right_on='efo_id')[['molecule_name', 'mondo_id']] efo_match = efo_match.rename(columns={"mondo_id": "disease_id"}) efo_match.drop_duplicates(inplace=True) + # logger.info(efo_match) cat_df = pd.concat([mondo_match, efo_match]) logger.info(cat_df.shape) @@ -78,7 +78,7 @@ def process(): cat_df.columns = col_names cat_df["source"] = cat_df["source"].str.upper() - create_import(df=cat_df, meta_id=meta_id) + create_import(df=cat_df, meta_id=args.name) if __name__ == "__main__": process() From fabd7d6804ab7bc8f2da5917de870687e660f71f Mon Sep 17 00:00:00 2001 From: Marina Vabistsevits Date: Thu, 2 Dec 2021 16:31:58 +0000 Subject: [PATCH 4/4] update local build guide --- ADDING_DATA_DETAILED_GUIDE.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ADDING_DATA_DETAILED_GUIDE.md b/ADDING_DATA_DETAILED_GUIDE.md index 611ecc1..219c537 100644 --- a/ADDING_DATA_DETAILED_GUIDE.md +++ b/ADDING_DATA_DETAILED_GUIDE.md @@ -56,7 +56,7 @@ There are 3 steps: ​ #### Prep * The graph build has to happen on jojo (or other server) -* (source bashrc) and conda activate neo4j_build +* (`source ~/.bashrc`) and `conda activate neo4j_build` * Make a folder in `workflow/source_data/FOLDER` * Modify `DATA_DIR` in `.env` when running the source script to point at the local source_data folder ​ @@ -66,19 +66,20 @@ There are 3 steps: snakemake -r clean_all -j 1 snakemake -r all -j 4 ``` +If there are issues with missing data, you will need to run scripts that make those datasets manually, to determine the issue (e.g. new data version has extra columns) and fix it to produce the missing data. ​ #### Step 2 ​ -Assuming scripts and ymls are created and locally testes, run: +Assuming scripts and ymls are created and locally tested, run: ​ ``` # run source script python -m workflow.scripts.source.SOURCE_SCRIPT ​ -# run processing script -python -m workflow.scripts.processing.rels.PROCESSING_SCRIPT -n (rel name in data_integration.yml) -d workflow/source_data/ +# run processing script (-d is optional if DATA_DIR in .env is set to a local path) +python -m workflow.scripts.processing.rels.PROCESSING_SCRIPT -n (name in data_integration.yml) -d workflow/source_data/ ​ -# check new data +# check new data (gives a short uninformative message as if nothing happened) snakemake -r check_new_data -j 10 ``` ​