From 75d79ff4341a2926e51fe79b8abf6b697c7f16a8 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Fri, 20 Mar 2026 14:05:34 +0100 Subject: [PATCH 01/25] implement run_ms2query --- ms2query/run_ms2query.py | 73 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 ms2query/run_ms2query.py diff --git a/ms2query/run_ms2query.py b/ms2query/run_ms2query.py new file mode 100644 index 0000000..0461882 --- /dev/null +++ b/ms2query/run_ms2query.py @@ -0,0 +1,73 @@ +import json +from pathlib import Path +from typing import Sequence, Tuple +import numpy as np +import pandas as pd +from matchms import Spectrum +from matchms.importing import load_spectra +from ms2deepscore.models import load_model +from ms2deepscore.vector_operations import cosine_similarity_matrix +from tqdm import tqdm +from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.benchmarking.Embeddings import Embeddings +from ms2query.benchmarking.Fingerprints import Fingerprints +from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores + + +def run_ms2query( + query_embeddings: Embeddings, + library_embeddings: Embeddings, + library_metadata: pd.DataFrame, + spectrum_indices_per_inchikey: dict[str, Tuple[int, ...]], + top_k_tanimoto_scores: TopKTanimotoScores, + batch_size: int = 1000, +): + num_of_query_embeddings = query_embeddings.embeddings.shape[0] + + library_index_highest_ms2deepscore = np.zeros((num_of_query_embeddings), dtype=int) + ms2query_scores = [] + for start_idx in tqdm( + range(0, num_of_query_embeddings, batch_size), + desc="Predicting highest ms2deepscore per batch of " + + str(min(batch_size, num_of_query_embeddings)) + + " embeddings", + ): + # Do MS2DeepScore predictions for batch + end_idx = min(start_idx + batch_size, num_of_query_embeddings) + selected_query_embeddings = query_embeddings.embeddings[start_idx:end_idx] + score_matrix = cosine_similarity_matrix(selected_query_embeddings, library_embeddings.embeddings) + highest_score_idx = np.argmax(score_matrix, axis=1) + library_index_highest_ms2deepscore[start_idx:end_idx] = highest_score_idx + + # get predicted inchikeys + predicted_inchikeys = library_metadata.iloc[highest_score_idx]["inchikey"] + # Compute MS2Query reliability score + ms2query_scores.extend( + get_ms2query_reliability_prediction( + predicted_inchikeys, spectrum_indices_per_inchikey, top_k_tanimoto_scores, score_matrix + ) + ) + + # construct results df + results = library_metadata.iloc[library_index_highest_ms2deepscore] + results["ms2query_reliability_prediction"] = ms2query_scores + return results + + +def get_ms2query_reliability_prediction( + predicted_inchikeys: list[str], + spectrum_indices_per_inchikey, + top_k_tanimoto_scores: TopKTanimotoScores, + ms2deepscore_score_matrix, +) -> list[float]: + ms2query_scores = [] + for query_spectrum_index, library_inchikey in enumerate(predicted_inchikeys): + top_k_inchikeys = top_k_tanimoto_scores.select_top_k_inchikeys(library_inchikey[:14]) + maximum_ms2deepscores = np.zeros(top_k_tanimoto_scores.k, dtype=float) + for i, inchikey in enumerate(top_k_inchikeys): + spectrum_indexes = spectrum_indices_per_inchikey[inchikey] + highest_ms2deepscore = np.max(ms2deepscore_score_matrix[query_spectrum_index, spectrum_indexes]) + maximum_ms2deepscores[i] = highest_ms2deepscore + ms2query_scores.append(np.mean(maximum_ms2deepscores)) + # todo get the spectrum hashes instead of the indexes for lookup later. + return ms2query_scores From a7bc22a2536ba9f065d6a9f6e8a96cacbce71aa8 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Fri, 20 Mar 2026 14:06:28 +0100 Subject: [PATCH 02/25] Add extract_metadata_from_library --- ms2query/run_ms2query.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ms2query/run_ms2query.py b/ms2query/run_ms2query.py index 0461882..e0cd763 100644 --- a/ms2query/run_ms2query.py +++ b/ms2query/run_ms2query.py @@ -71,3 +71,13 @@ def get_ms2query_reliability_prediction( ms2query_scores.append(np.mean(maximum_ms2deepscores)) # todo get the spectrum hashes instead of the indexes for lookup later. return ms2query_scores + + +def extract_metadata_from_library(spectra: AnnotatedSpectrumSet, metadata_to_collect: list): + collected_metadata = {key: [] for key in metadata_to_collect} + collected_metadata["spectrum_hashes"] = [] + for spectrum in tqdm(spectra.spectra, desc="Extracting metadata df from spectra"): + for metadata_key in metadata_to_collect: + collected_metadata[metadata_key].append(spectrum.get(metadata_key)) + collected_metadata["spectrum_hashes"].append(spectrum.__hash__()) + return pd.DataFrame(collected_metadata) From 29cc1e1a7aa39d6d9f0464df2ff66727ab8a8f54 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Fri, 20 Mar 2026 14:07:06 +0100 Subject: [PATCH 03/25] Add first test run ms2query --- tests/test_run_ms2query.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/test_run_ms2query.py diff --git a/tests/test_run_ms2query.py b/tests/test_run_ms2query.py new file mode 100644 index 0000000..acc7ab8 --- /dev/null +++ b/tests/test_run_ms2query.py @@ -0,0 +1,34 @@ +from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.benchmarking.Fingerprints import Fingerprints +from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores +from ms2query.run_ms2query import extract_metadata_from_library, run_ms2query +from tests.helper_functions import create_test_spectra, ms2deepscore_model + + +def test_run_ms2query(): + model = ms2deepscore_model() + library_spectra = AnnotatedSpectrumSet.create_spectrum_set(create_test_spectra(nr_of_inchikeys=7)) + test_spectra = AnnotatedSpectrumSet.create_spectrum_set(create_test_spectra(1, nr_of_inchikeys=3)) + library_spectra.add_embeddings(model) + test_spectra.add_embeddings(model) + fingerprints = Fingerprints.from_spectrum_set(library_spectra, "daylight", 100) + top_k_tanimoto_scores = TopKTanimotoScores.calculate_from_fingerprints(fingerprints, fingerprints, 3) + spectrum_indices_per_inchikey = library_spectra.spectrum_indices_per_inchikey + metadata_library = extract_metadata_from_library( + library_spectra, + [ + "precursor_mz", + "collision_energy", + "compound_name", + "smiles", + "inchikey", + ], + ) + results = run_ms2query( + test_spectra.embeddings, + library_spectra.embeddings, + metadata_library, + spectrum_indices_per_inchikey, + top_k_tanimoto_scores, + ) + print(results) From fb0107d1f72c752a9ee424089767f13f386b9399 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Fri, 20 Mar 2026 14:19:16 +0100 Subject: [PATCH 04/25] Add save and load options for top_k_tanimoto_scores --- ms2query/benchmarking/TopKTanimotoScores.py | 38 +++++++++++++- pyproject.toml | 1 + .../test_top_k_tanimoto_scores.py | 52 +++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/ms2query/benchmarking/TopKTanimotoScores.py b/ms2query/benchmarking/TopKTanimotoScores.py index 07c0912..cd57271 100644 --- a/ms2query/benchmarking/TopKTanimotoScores.py +++ b/ms2query/benchmarking/TopKTanimotoScores.py @@ -1,3 +1,4 @@ +from pathlib import Path import numpy as np import pandas as pd from ms2query.benchmarking.Fingerprints import Fingerprints @@ -27,13 +28,21 @@ def _create_multi_index( combined_data = np.empty((len(inchikey_indexes), self.k * 2), dtype=object) combined_data[:, 0::2] = top_k_inchikeys combined_data[:, 1::2] = tanimoto_scores_for_top_k - return pd.DataFrame(combined_data, index=inchikey_indexes, columns=columns) + df = pd.DataFrame(combined_data, index=inchikey_indexes, columns=columns) + + # Cast score columns to float64 + score_cols = [(rank, "score") for rank in [f"Rank_{i + 1}" for i in range(self.k)]] + df[score_cols] = df[score_cols].astype(float) + + return df @classmethod def calculate_from_fingerprints(cls, query_fingerprints: Fingerprints, target_fingerprints: Fingerprints, k): """ Gets the top k highest inchikeys and scores for each inchikey in query_fingerprints from target_fingerprints """ + if target_fingerprints.fingerprints.shape[0] < k: + raise ValueError("K cannot be larger than the number of fingerprints") similarity_scores = generalized_tanimoto_similarity_matrix( query_fingerprints.fingerprints, target_fingerprints.fingerprints ) @@ -67,3 +76,30 @@ def get_all_average_tanimoto_scores(self) -> dict[str, float]: average_per_inchikey_df = scores_df.mean(axis=1) return average_per_inchikey_df.to_dict() + + def save(self, path: str | Path) -> None: + """Save the TopKTanimotoScores to disk as a parquet file. + + Args: + path: File path without extension, e.g. "/data/top_k_scores". + """ + Path(path).with_suffix(".parquet").parent.mkdir(parents=True, exist_ok=True) + self.top_k_inchikeys_and_scores.to_parquet(Path(path).with_suffix(".parquet")) + + @classmethod + def load(cls, path: str | Path) -> "TopKTanimotoScores": + """Load a previously saved TopKTanimotoScores from disk. + + Args: + path: File path without extension, e.g. "/data/top_k_scores". + + Returns: + A fully reconstructed TopKTanimotoScores instance. + """ + df = pd.read_parquet(Path(path).with_suffix(".parquet")) + df.columns.names = ["result_rank", "attribute"] + + instance = cls.__new__(cls) + instance.k = len(df.columns.get_level_values("result_rank").unique()) + instance.top_k_inchikeys_and_scores = df + return instance diff --git a/pyproject.toml b/pyproject.toml index fbaac97..b416b53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ ms2deepscore= ">=2.6.0" rdkit= ">2024.3.4" nmslib= ">=2.0.0" umap-learn= ">=0.5.7" +pyarrow= ">=14.0.1" [tool.poetry.group.dev.dependencies] decorator = "^5.1.1" diff --git a/tests/test_benchmarking/test_top_k_tanimoto_scores.py b/tests/test_benchmarking/test_top_k_tanimoto_scores.py index 2c8ecd2..bff7ecc 100644 --- a/tests/test_benchmarking/test_top_k_tanimoto_scores.py +++ b/tests/test_benchmarking/test_top_k_tanimoto_scores.py @@ -1,4 +1,5 @@ import numpy as np +import pandas as pd import pytest from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores from tests.helper_functions import make_test_fingerprints @@ -26,3 +27,54 @@ def test_calculate_from_fingerprints(): "AAAAAAAAAAAAAD": 0.75, "AAAAAAAAAAAAAE": 1.0, } + + +@pytest.fixture +def sample_scores(): + """Creates a simple TopKTanimotoScores instance for testing.""" + tanimoto_scores = np.array( + [ + [0.9, 0.7, 0.5], + [0.8, 0.6, 0.4], + [0.95, 0.85, 0.75], + ] + ) + top_k_inchikeys = np.array( + [ + ["INCHI_A", "INCHI_B", "INCHI_C"], + ["INCHI_B", "INCHI_C", "INCHI_A"], + ["INCHI_C", "INCHI_A", "INCHI_B"], + ] + ) + inchikey_indexes = np.array(["QUERY_1", "QUERY_2", "QUERY_3"]) + return TopKTanimotoScores(tanimoto_scores, top_k_inchikeys, inchikey_indexes) + + +# ----- save and load tests ----- +def test_save_creates_parquet_file(sample_scores, tmp_path): + sample_scores.save(tmp_path / "test_scores") + assert (tmp_path / "test_scores.parquet").exists() + + +def test_save_creates_parent_directories(sample_scores, tmp_path): + sample_scores.save(tmp_path / "nested" / "dir" / "test_scores") + assert (tmp_path / "nested" / "dir" / "test_scores.parquet").exists() + + +def test_roundtrip_produces_identical_object(sample_scores, tmp_path): + sample_scores.save(tmp_path / "test_scores") + loaded = TopKTanimotoScores.load(tmp_path / "test_scores") + + assert loaded.k == sample_scores.k + pd.testing.assert_frame_equal(loaded.top_k_inchikeys_and_scores, sample_scores.top_k_inchikeys_and_scores) + assert sample_scores.select_top_k_inchikeys_and_scores("QUERY_1") == loaded.select_top_k_inchikeys_and_scores( + "QUERY_1" + ) + assert sample_scores.select_top_k_inchikeys("QUERY_2") == loaded.select_top_k_inchikeys("QUERY_2") + assert sample_scores.select_average_score("QUERY_3") == pytest.approx(loaded.select_average_score("QUERY_3")) + + +def test_roundtrip_accepts_string_path(sample_scores, tmp_path): + sample_scores.save(str(tmp_path / "test_scores")) + loaded = TopKTanimotoScores.load(str(tmp_path / "test_scores")) + pd.testing.assert_frame_equal(loaded.top_k_inchikeys_and_scores, sample_scores.top_k_inchikeys_and_scores) From 711bd024c301ff3e07af61a0bbc082ef4ba8a304 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Fri, 20 Mar 2026 14:35:03 +0100 Subject: [PATCH 05/25] Make create_ms2query_library method --- ms2query/run_ms2query.py | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/ms2query/run_ms2query.py b/ms2query/run_ms2query.py index e0cd763..91968ba 100644 --- a/ms2query/run_ms2query.py +++ b/ms2query/run_ms2query.py @@ -73,6 +73,47 @@ def get_ms2query_reliability_prediction( return ms2query_scores +def create_ms2query_library(library_spectra_file: str, ms2deepscore_model_file_name: str): + """Loads in a library and saves the embeddings and top_k_tanimoto_scores""" + spectrum_file_directory = Path("/some/dir/file.txt").parent + embedding_file_location = spectrum_file_directory / "embeddings.npz" + top_k_tanimoto_score_file_location = spectrum_file_directory / "top_k_tanimoto_scores.parquet" + reference_metadata_file = spectrum_file_directory / "library_metadata.parquet" + if embedding_file_location.exists(): + raise FileExistsError("There is already an embedding.npy file in the directory of your library spectra") + if top_k_tanimoto_score_file_location.exists(): + raise FileExistsError( + "There is already an top_k_tanimoto_scores.parquet file in the directory of your library spectra" + ) + + library_spectra = list(tqdm(load_spectra(library_spectra_file), "Loading library spectra")) + library_spectra = AnnotatedSpectrumSet.create_spectrum_set(library_spectra) + ms2deepscore_model = load_model(ms2deepscore_model_file_name) + library_spectra.add_embeddings(ms2deepscore_model) + + library_spectra._embeddings.save(embedding_file_location) + + fingerprints = Fingerprints.from_spectrum_set(library_spectra, "daylight", 4096) + top_k_tanimoto_scores = TopKTanimotoScores.calculate_from_fingerprints( + fingerprints, + fingerprints, + k=8, + ) + top_k_tanimoto_scores.save(top_k_tanimoto_score_file_location) + reference_metadata = extract_metadata_from_library( + library_spectra, + [ + "precursor_mz", + "retention_time", + "collision_energy", + "compound_name", + "smiles", + "inchikey", + ], + ) + reference_metadata.to_parquet(reference_metadata_file) + + def extract_metadata_from_library(spectra: AnnotatedSpectrumSet, metadata_to_collect: list): collected_metadata = {key: [] for key in metadata_to_collect} collected_metadata["spectrum_hashes"] = [] From 931d1cbc5b4d3fa394c3efbbbe48f34e5978e709 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Fri, 20 Mar 2026 14:35:20 +0100 Subject: [PATCH 06/25] add run_ms2query_from_files --- ms2query/run_ms2query.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/ms2query/run_ms2query.py b/ms2query/run_ms2query.py index 91968ba..0e6e282 100644 --- a/ms2query/run_ms2query.py +++ b/ms2query/run_ms2query.py @@ -1,4 +1,5 @@ import json +from collections import defaultdict from pathlib import Path from typing import Sequence, Tuple import numpy as np @@ -14,11 +15,34 @@ from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores +def run_ms2query_from_files( + query_spectrum_file, + ms2deepscore_model_file_name, + reference_embeddings_file, + top_k_tanimoto_scores_file, + reference_metadata_file, +): + reference_embeddings = Embeddings.load(reference_embeddings_file) + top_k_tanimoto_scores = TopKTanimotoScores.load(top_k_tanimoto_scores_file) + reference_metadata = pd.read_parquet(reference_metadata_file) + # Get the spectrum_indices_per_inchikey + spectrum_indices_per_inchikey = defaultdict(list) + for lib_spec_index, inchikey in enumerate(reference_metadata["inchikey"]): + spectrum_indices_per_inchikey[inchikey[:14]].append(lib_spec_index) + + query_spectra = list(tqdm(load_spectra(query_spectrum_file), desc="loading_in_query_spectra")) + ms2deepscore_model = load_model(ms2deepscore_model_file_name) + query_embeddings = Embeddings.create_from_spectra(query_spectra, ms2deepscore_model) + run_ms2query( + query_embeddings, reference_embeddings, reference_metadata, spectrum_indices_per_inchikey, top_k_tanimoto_scores + ) + + def run_ms2query( query_embeddings: Embeddings, library_embeddings: Embeddings, library_metadata: pd.DataFrame, - spectrum_indices_per_inchikey: dict[str, Tuple[int, ...]], + spectrum_indices_per_inchikey: defaultdict[str, list[int]], top_k_tanimoto_scores: TopKTanimotoScores, batch_size: int = 1000, ): From be5e756d518096690d4bac9c2c0e332714e1bab0 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 12:18:53 +0100 Subject: [PATCH 07/25] Add some extra inchikey examples --- tests/helper_functions.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/helper_functions.py b/tests/helper_functions.py index 82d1709..875ae13 100644 --- a/tests/helper_functions.py +++ b/tests/helper_functions.py @@ -87,6 +87,36 @@ def get_inchikey_inchi_pairs(number_of_pairs): "CC(C)C[C@@H](C(=O)O)N", "L-Leucine", ), + ( + "BSYNRYMUTXBXSQ-UHFFFAOYSA-N", + "InChI=1S/C9H8O4/c1-6(10)13-8-5-3-2-4-7(8)9(11)12/h2-5H,1H3,(H,11,12)", + "CC(=O)OC1=CC=CC=C1C(=O)O", + "Aspirin", + ), + ( + "WHUUTDBJXJRKMK-VKHMYHEASA-N", + "InChI=1S/C5H9NO4/c6-3(5(9)10)1-2-4(7)8/h3H,1-2,6H2,(H,7,8)(H,9,10)/t3-/m0/s1", + "C(CC(=O)O)[C@@H](C(=O)O)N", + "L-Glutamic acid", + ), + ( + "ZKHQWZAMYRWXGA-KQYNXXCUSA-N", + "InChI=1S/C10H14N5O7P/c11-8-5-9(13-2-12-8)15(3-14-5)10-7(17)6(16)4(22-10)1-21-23(18,19)20/h2-4,6-7,10,16-17H,1H2,(H2,11,12,13)(H2,18,19,20)/t4-,6-,7-,10-/m1/s1", + "C1=NC(=C2C(=N1)N(C=N2)[C@H]3[C@@H]([C@@H]([C@H](O3)COP(=O)(O)O)O)O)N", + "AMP", + ), + ( + "GVJHHUAWPYXKBD-UHFFFAOYSA-N", + "InChI=1S/C2H6O/c1-2-3/h3H,2H2,1H3", + "CCO", + "Ethanol", + ), + ( + "IKHGUXGNUITLKF-XPULMUKRSA-N", + "InChI=1S/C9H13NO3/c1-6(11)13-8-4-2-7(3-5-8)9(10)12/h2-6,11H,1H3,(H2,10,12)/t6-/m0/s1", + "C[C@@H](C1=CC=C(C=C1)C(=O)N)O", + "Salbutamol", + ), ) if number_of_pairs > len(inchikey_inchi_pairs): raise ValueError("Not enough example compounds, add some in conftest") From 0cb5a530c394f3e5752e1da137906e601de4bbe1 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 12:19:34 +0100 Subject: [PATCH 08/25] Convert run_ms2query to class --- ms2query/run_ms2query.py | 268 +++++++++++++++++++++++-------------- tests/test_run_ms2query.py | 50 +++++-- 2 files changed, 208 insertions(+), 110 deletions(-) diff --git a/ms2query/run_ms2query.py b/ms2query/run_ms2query.py index 0e6e282..080dc03 100644 --- a/ms2query/run_ms2query.py +++ b/ms2query/run_ms2query.py @@ -1,81 +1,192 @@ -import json from collections import defaultdict from pathlib import Path -from typing import Sequence, Tuple +from typing import Sequence import numpy as np import pandas as pd -from matchms import Spectrum from matchms.importing import load_spectra -from ms2deepscore.models import load_model +from matchms.Spectrum import Spectrum +from ms2deepscore.models import SiameseSpectralModel, load_model from ms2deepscore.vector_operations import cosine_similarity_matrix from tqdm import tqdm from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.benchmarking.Embeddings import Embeddings +from ms2query.benchmarking.Embeddings import Embeddings, _to_json_serializable from ms2query.benchmarking.Fingerprints import Fingerprints from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores +class MS2QueryLibrary: + # Set default file names to enable save and load per library + embedding_file_name = "embeddings.npz" + top_k_tanimoto_scores_file_name = "top_k_tanimoto_scores.parquet" + reference_metadata_file_name = "library_metadata.parquet" + ms2deepscore_model_file_name = "ms2deepscore_model.pt" + metadata_to_store = [ + "precursor_mz", + "retention_time", + "collision_energy", + "compound_name", + "smiles", + "inchikey", + ] + fingerprint_type = "daylight" + fingerprint_nbits = 4096 + top_k_inchikeys = 8 + + def __init__( + self, + ms2deepscore_model: SiameseSpectralModel, + reference_embeddings: Embeddings, + top_k_tanimoto_scores: TopKTanimotoScores, + reference_metadata: pd.DataFrame, + ): + self.ms2deepscore_model = ms2deepscore_model + self.reference_embeddings = reference_embeddings + self.top_k_tanimoto_scores = top_k_tanimoto_scores + self.reference_metadata = reference_metadata + + # Check that the loaded files match + if _to_json_serializable(ms2deepscore_model.model_settings.get_dict()) != reference_embeddings.model_settings: + raise ValueError( + "The settings of the ms2deepscore model does not match the model used for creating the library embeddings" + ) + if list(self.reference_metadata["spectrum_hashes"]) != [ + str(spectrum_hash) for spectrum_hash in reference_embeddings.index_to_spectrum_hash + ]: + raise ValueError("The loaded metadata does not match the used embeddings") + if {inchikey[:14] for inchikey in reference_metadata["inchikey"]} != set( + top_k_tanimoto_scores.top_k_inchikeys_and_scores.index + ): + raise ValueError("The inchikeys in the metadata and in the top_k_tanimoto_scores do not match") + + # Get the spectrum_indices_per_inchikey + self.spectrum_indices_per_inchikey = defaultdict(list) + for lib_spec_index, inchikey in enumerate(reference_metadata["inchikey"]): + self.spectrum_indices_per_inchikey[inchikey[:14]].append(lib_spec_index) + + @classmethod + def load_from_directory(cls, library_file_directory) -> "MS2QueryLibrary": + reference_embeddings_file = library_file_directory / cls.embedding_file_name + top_k_tanimoto_scores_file = library_file_directory / cls.top_k_tanimoto_scores_file_name + reference_metadata_file = library_file_directory / cls.reference_metadata_file_name + ms2deepscore_model_file_name = library_file_directory = cls.ms2deepscore_model_file_name + return cls.load_from_files( + ms2deepscore_model_file_name, reference_embeddings_file, top_k_tanimoto_scores_file, reference_metadata_file + ) + + @classmethod + def load_from_files( + cls, + ms2deepscore_model_file_name, + reference_embeddings_file, + top_k_tanimoto_scores_file, + reference_metadata_file, + ) -> "MS2QueryLibrary": + return cls( + load_model(ms2deepscore_model_file_name), + Embeddings.load(reference_embeddings_file), + TopKTanimotoScores.load(top_k_tanimoto_scores_file), + pd.read_parquet(reference_metadata_file), + ) + + @classmethod + def create_from_spectra( + cls, + library_spectra: Sequence[Spectrum], + ms2deepscore_model_file_name: str, + store_file_directory=None, + store_files=True, + ) -> "MS2QueryLibrary": + """Creates all the files needed for MS2Query and stores them""" + if store_file_directory is None: + store_file_directory = Path(ms2deepscore_model_file_name).parent + if store_files: + # Check the files don't exist yet + for file in ( + store_file_directory / cls.embedding_file_name, + store_file_directory / cls.top_k_tanimoto_scores_file_name, + store_file_directory / cls.reference_metadata_file_name, + ): + if file.exists(): + raise FileExistsError(f"There is already a file stored with the name {file}") + + # library_spectra = list(tqdm(load_spectra(library_spectra_file), "Loading library spectra")) + library_spectrum_set = AnnotatedSpectrumSet.create_spectrum_set(library_spectra) + ms2deepscore_model = load_model(ms2deepscore_model_file_name) + library_spectrum_set.add_embeddings(ms2deepscore_model) + + fingerprints = Fingerprints.from_spectrum_set(library_spectrum_set, cls.fingerprint_type, cls.fingerprint_nbits) + top_k_tanimoto_scores = TopKTanimotoScores.calculate_from_fingerprints( + fingerprints, fingerprints, cls.top_k_inchikeys + ) + reference_metadata = extract_metadata_from_library( + library_spectrum_set, + cls.metadata_to_store, + ) + + if store_files: + reference_metadata.to_parquet(store_file_directory / cls.reference_metadata_file_name) + top_k_tanimoto_scores.save(store_file_directory / cls.top_k_tanimoto_scores_file_name) + library_spectrum_set.embeddings.save(store_file_directory / cls.embedding_file_name) + return cls(ms2deepscore_model, library_spectrum_set.embeddings, top_k_tanimoto_scores, reference_metadata) + + def run_ms2query( + self, + query_spectra: Sequence[Spectrum], + batch_size: int = 1000, + ) -> pd.DataFrame: + + query_embeddings = Embeddings.create_from_spectra(query_spectra, self.ms2deepscore_model) + + num_of_query_embeddings = query_embeddings.embeddings.shape[0] + + library_index_highest_ms2deepscore = np.zeros((num_of_query_embeddings), dtype=int) + ms2query_scores = [] + for start_idx in tqdm( + range(0, num_of_query_embeddings, batch_size), + desc="Predicting highest ms2deepscore per batch of " + + str(min(batch_size, num_of_query_embeddings)) + + " embeddings", + ): + # Do MS2DeepScore predictions for batch + end_idx = min(start_idx + batch_size, num_of_query_embeddings) + selected_query_embeddings = query_embeddings.embeddings[start_idx:end_idx] + score_matrix = cosine_similarity_matrix(selected_query_embeddings, self.reference_embeddings.embeddings) + highest_score_idx = np.argmax(score_matrix, axis=1) + library_index_highest_ms2deepscore[start_idx:end_idx] = highest_score_idx + + # get predicted inchikeys + predicted_inchikeys = self.reference_metadata.iloc[highest_score_idx]["inchikey"] + # Compute MS2Query reliability score + ms2query_scores.extend( + get_ms2query_reliability_prediction( + predicted_inchikeys, self.spectrum_indices_per_inchikey, self.top_k_tanimoto_scores, score_matrix + ) + ) + + # construct results df + results = self.reference_metadata.iloc[library_index_highest_ms2deepscore] + results["ms2query_reliability_prediction"] = ms2query_scores + return results + + def run_ms2query_from_files( query_spectrum_file, ms2deepscore_model_file_name, reference_embeddings_file, top_k_tanimoto_scores_file, reference_metadata_file, + save_file_location, ): - reference_embeddings = Embeddings.load(reference_embeddings_file) - top_k_tanimoto_scores = TopKTanimotoScores.load(top_k_tanimoto_scores_file) - reference_metadata = pd.read_parquet(reference_metadata_file) - # Get the spectrum_indices_per_inchikey - spectrum_indices_per_inchikey = defaultdict(list) - for lib_spec_index, inchikey in enumerate(reference_metadata["inchikey"]): - spectrum_indices_per_inchikey[inchikey[:14]].append(lib_spec_index) - - query_spectra = list(tqdm(load_spectra(query_spectrum_file), desc="loading_in_query_spectra")) - ms2deepscore_model = load_model(ms2deepscore_model_file_name) - query_embeddings = Embeddings.create_from_spectra(query_spectra, ms2deepscore_model) - run_ms2query( - query_embeddings, reference_embeddings, reference_metadata, spectrum_indices_per_inchikey, top_k_tanimoto_scores + ms2query_library = MS2QueryLibrary.load_from_files( + ms2deepscore_model_file_name, + reference_embeddings_file, + top_k_tanimoto_scores_file, + reference_metadata_file, ) - -def run_ms2query( - query_embeddings: Embeddings, - library_embeddings: Embeddings, - library_metadata: pd.DataFrame, - spectrum_indices_per_inchikey: defaultdict[str, list[int]], - top_k_tanimoto_scores: TopKTanimotoScores, - batch_size: int = 1000, -): - num_of_query_embeddings = query_embeddings.embeddings.shape[0] - - library_index_highest_ms2deepscore = np.zeros((num_of_query_embeddings), dtype=int) - ms2query_scores = [] - for start_idx in tqdm( - range(0, num_of_query_embeddings, batch_size), - desc="Predicting highest ms2deepscore per batch of " - + str(min(batch_size, num_of_query_embeddings)) - + " embeddings", - ): - # Do MS2DeepScore predictions for batch - end_idx = min(start_idx + batch_size, num_of_query_embeddings) - selected_query_embeddings = query_embeddings.embeddings[start_idx:end_idx] - score_matrix = cosine_similarity_matrix(selected_query_embeddings, library_embeddings.embeddings) - highest_score_idx = np.argmax(score_matrix, axis=1) - library_index_highest_ms2deepscore[start_idx:end_idx] = highest_score_idx - - # get predicted inchikeys - predicted_inchikeys = library_metadata.iloc[highest_score_idx]["inchikey"] - # Compute MS2Query reliability score - ms2query_scores.extend( - get_ms2query_reliability_prediction( - predicted_inchikeys, spectrum_indices_per_inchikey, top_k_tanimoto_scores, score_matrix - ) - ) - - # construct results df - results = library_metadata.iloc[library_index_highest_ms2deepscore] - results["ms2query_reliability_prediction"] = ms2query_scores - return results + query_spectra = list(tqdm(load_spectra(query_spectrum_file), desc="loading_in_query_spectra")) + results_df = ms2query_library.run_ms2query(query_spectra) + results_df.to_csv(save_file_location) def get_ms2query_reliability_prediction( @@ -97,52 +208,11 @@ def get_ms2query_reliability_prediction( return ms2query_scores -def create_ms2query_library(library_spectra_file: str, ms2deepscore_model_file_name: str): - """Loads in a library and saves the embeddings and top_k_tanimoto_scores""" - spectrum_file_directory = Path("/some/dir/file.txt").parent - embedding_file_location = spectrum_file_directory / "embeddings.npz" - top_k_tanimoto_score_file_location = spectrum_file_directory / "top_k_tanimoto_scores.parquet" - reference_metadata_file = spectrum_file_directory / "library_metadata.parquet" - if embedding_file_location.exists(): - raise FileExistsError("There is already an embedding.npy file in the directory of your library spectra") - if top_k_tanimoto_score_file_location.exists(): - raise FileExistsError( - "There is already an top_k_tanimoto_scores.parquet file in the directory of your library spectra" - ) - - library_spectra = list(tqdm(load_spectra(library_spectra_file), "Loading library spectra")) - library_spectra = AnnotatedSpectrumSet.create_spectrum_set(library_spectra) - ms2deepscore_model = load_model(ms2deepscore_model_file_name) - library_spectra.add_embeddings(ms2deepscore_model) - - library_spectra._embeddings.save(embedding_file_location) - - fingerprints = Fingerprints.from_spectrum_set(library_spectra, "daylight", 4096) - top_k_tanimoto_scores = TopKTanimotoScores.calculate_from_fingerprints( - fingerprints, - fingerprints, - k=8, - ) - top_k_tanimoto_scores.save(top_k_tanimoto_score_file_location) - reference_metadata = extract_metadata_from_library( - library_spectra, - [ - "precursor_mz", - "retention_time", - "collision_energy", - "compound_name", - "smiles", - "inchikey", - ], - ) - reference_metadata.to_parquet(reference_metadata_file) - - def extract_metadata_from_library(spectra: AnnotatedSpectrumSet, metadata_to_collect: list): collected_metadata = {key: [] for key in metadata_to_collect} collected_metadata["spectrum_hashes"] = [] for spectrum in tqdm(spectra.spectra, desc="Extracting metadata df from spectra"): for metadata_key in metadata_to_collect: collected_metadata[metadata_key].append(spectrum.get(metadata_key)) - collected_metadata["spectrum_hashes"].append(spectrum.__hash__()) + collected_metadata["spectrum_hashes"].append(str(spectrum.__hash__())) return pd.DataFrame(collected_metadata) diff --git a/tests/test_run_ms2query.py b/tests/test_run_ms2query.py index acc7ab8..df48abf 100644 --- a/tests/test_run_ms2query.py +++ b/tests/test_run_ms2query.py @@ -1,19 +1,22 @@ +import os + +import pandas as pd + from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet from ms2query.benchmarking.Fingerprints import Fingerprints from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores -from ms2query.run_ms2query import extract_metadata_from_library, run_ms2query -from tests.helper_functions import create_test_spectra, ms2deepscore_model +from ms2query.run_ms2query import extract_metadata_from_library, run_ms2query_from_files, MS2QueryLibrary +from tests.helper_functions import TEST_RESOURCES_PATH, create_test_spectra, ms2deepscore_model +from matchms.exporting import save_as_mgf def test_run_ms2query(): model = ms2deepscore_model() library_spectra = AnnotatedSpectrumSet.create_spectrum_set(create_test_spectra(nr_of_inchikeys=7)) - test_spectra = AnnotatedSpectrumSet.create_spectrum_set(create_test_spectra(1, nr_of_inchikeys=3)) + test_spectra = create_test_spectra(1, nr_of_inchikeys=3) library_spectra.add_embeddings(model) - test_spectra.add_embeddings(model) fingerprints = Fingerprints.from_spectrum_set(library_spectra, "daylight", 100) top_k_tanimoto_scores = TopKTanimotoScores.calculate_from_fingerprints(fingerprints, fingerprints, 3) - spectrum_indices_per_inchikey = library_spectra.spectrum_indices_per_inchikey metadata_library = extract_metadata_from_library( library_spectra, [ @@ -24,11 +27,36 @@ def test_run_ms2query(): "inchikey", ], ) - results = run_ms2query( - test_spectra.embeddings, - library_spectra.embeddings, - metadata_library, - spectrum_indices_per_inchikey, - top_k_tanimoto_scores, + + results = MS2QueryLibrary(model, library_spectra.embeddings, top_k_tanimoto_scores, metadata_library).run_ms2query( + test_spectra ) print(results) + + +def test_create_library(tmp_path): + lib_spectra = create_test_spectra(nr_of_inchikeys=10, number_of_spectra_per_inchikey=3) + # save_as_mgf(lib_spectra, os.path.join(tmp_path, "library_spectra.mgf")) + ms2deepscore_model_file = os.path.join(TEST_RESOURCES_PATH, "ms2deepscore_testmodel_v1.pt") + MS2QueryLibrary.create_from_spectra(lib_spectra, ms2deepscore_model_file, tmp_path) + assert (tmp_path / MS2QueryLibrary.embedding_file_name).exists() + assert (tmp_path / MS2QueryLibrary.top_k_tanimoto_scores_file_name).exists() + assert (tmp_path / MS2QueryLibrary.reference_metadata_file_name).exists() + + +def test_create_and_use_library(tmp_path): + lib_spectra = create_test_spectra(nr_of_inchikeys=10, number_of_spectra_per_inchikey=3) + ms2deepscore_model_file = os.path.join(TEST_RESOURCES_PATH, "ms2deepscore_testmodel_v1.pt") + ms2query_library = MS2QueryLibrary.create_from_spectra(lib_spectra, ms2deepscore_model_file, tmp_path) + test_spectra = create_test_spectra(1, nr_of_inchikeys=3) + results = ms2query_library.run_ms2query(test_spectra) + + ms2query_library_2 = MS2QueryLibrary.load_from_files( + ms2deepscore_model_file, + tmp_path / MS2QueryLibrary.embedding_file_name, + tmp_path / MS2QueryLibrary.top_k_tanimoto_scores_file_name, + tmp_path / MS2QueryLibrary.reference_metadata_file_name, + ) + + results_2 = ms2query_library_2.run_ms2query(test_spectra) + pd.testing.assert_frame_equal(results, results_2) From 3601e157631365c67a62c3cd43017a0fb01cad11 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 13:25:24 +0100 Subject: [PATCH 09/25] linting --- tests/test_run_ms2query.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_run_ms2query.py b/tests/test_run_ms2query.py index df48abf..8fe256e 100644 --- a/tests/test_run_ms2query.py +++ b/tests/test_run_ms2query.py @@ -1,13 +1,10 @@ import os - import pandas as pd - from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet from ms2query.benchmarking.Fingerprints import Fingerprints from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores -from ms2query.run_ms2query import extract_metadata_from_library, run_ms2query_from_files, MS2QueryLibrary +from ms2query.run_ms2query import MS2QueryLibrary, extract_metadata_from_library, run_ms2query_from_files from tests.helper_functions import TEST_RESOURCES_PATH, create_test_spectra, ms2deepscore_model -from matchms.exporting import save_as_mgf def test_run_ms2query(): From de0231583df899772591b6d4b6df8f9fea8b02d9 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 13:29:50 +0100 Subject: [PATCH 10/25] Rename and move MS2QueryLibrary to ReferenceLibrary --- .../ReferenceLibrary.py} | 10 +++---- tests/test_run_ms2query.py | 26 +++++++++++-------- 2 files changed, 20 insertions(+), 16 deletions(-) rename ms2query/{run_ms2query.py => benchmarking/ReferenceLibrary.py} (95%) diff --git a/ms2query/run_ms2query.py b/ms2query/benchmarking/ReferenceLibrary.py similarity index 95% rename from ms2query/run_ms2query.py rename to ms2query/benchmarking/ReferenceLibrary.py index 080dc03..35ad4c7 100644 --- a/ms2query/run_ms2query.py +++ b/ms2query/benchmarking/ReferenceLibrary.py @@ -14,7 +14,7 @@ from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores -class MS2QueryLibrary: +class ReferenceLibrary: # Set default file names to enable save and load per library embedding_file_name = "embeddings.npz" top_k_tanimoto_scores_file_name = "top_k_tanimoto_scores.parquet" @@ -64,7 +64,7 @@ def __init__( self.spectrum_indices_per_inchikey[inchikey[:14]].append(lib_spec_index) @classmethod - def load_from_directory(cls, library_file_directory) -> "MS2QueryLibrary": + def load_from_directory(cls, library_file_directory) -> "ReferenceLibrary": reference_embeddings_file = library_file_directory / cls.embedding_file_name top_k_tanimoto_scores_file = library_file_directory / cls.top_k_tanimoto_scores_file_name reference_metadata_file = library_file_directory / cls.reference_metadata_file_name @@ -80,7 +80,7 @@ def load_from_files( reference_embeddings_file, top_k_tanimoto_scores_file, reference_metadata_file, - ) -> "MS2QueryLibrary": + ) -> "ReferenceLibrary": return cls( load_model(ms2deepscore_model_file_name), Embeddings.load(reference_embeddings_file), @@ -95,7 +95,7 @@ def create_from_spectra( ms2deepscore_model_file_name: str, store_file_directory=None, store_files=True, - ) -> "MS2QueryLibrary": + ) -> "ReferenceLibrary": """Creates all the files needed for MS2Query and stores them""" if store_file_directory is None: store_file_directory = Path(ms2deepscore_model_file_name).parent @@ -177,7 +177,7 @@ def run_ms2query_from_files( reference_metadata_file, save_file_location, ): - ms2query_library = MS2QueryLibrary.load_from_files( + ms2query_library = ReferenceLibrary.load_from_files( ms2deepscore_model_file_name, reference_embeddings_file, top_k_tanimoto_scores_file, diff --git a/tests/test_run_ms2query.py b/tests/test_run_ms2query.py index 8fe256e..63714cc 100644 --- a/tests/test_run_ms2query.py +++ b/tests/test_run_ms2query.py @@ -3,7 +3,11 @@ from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet from ms2query.benchmarking.Fingerprints import Fingerprints from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores -from ms2query.run_ms2query import MS2QueryLibrary, extract_metadata_from_library, run_ms2query_from_files +from ms2query.benchmarking.ReferenceLibrary import ( + ReferenceLibrary, + extract_metadata_from_library, + run_ms2query_from_files, +) from tests.helper_functions import TEST_RESOURCES_PATH, create_test_spectra, ms2deepscore_model @@ -25,7 +29,7 @@ def test_run_ms2query(): ], ) - results = MS2QueryLibrary(model, library_spectra.embeddings, top_k_tanimoto_scores, metadata_library).run_ms2query( + results = ReferenceLibrary(model, library_spectra.embeddings, top_k_tanimoto_scores, metadata_library).run_ms2query( test_spectra ) print(results) @@ -35,24 +39,24 @@ def test_create_library(tmp_path): lib_spectra = create_test_spectra(nr_of_inchikeys=10, number_of_spectra_per_inchikey=3) # save_as_mgf(lib_spectra, os.path.join(tmp_path, "library_spectra.mgf")) ms2deepscore_model_file = os.path.join(TEST_RESOURCES_PATH, "ms2deepscore_testmodel_v1.pt") - MS2QueryLibrary.create_from_spectra(lib_spectra, ms2deepscore_model_file, tmp_path) - assert (tmp_path / MS2QueryLibrary.embedding_file_name).exists() - assert (tmp_path / MS2QueryLibrary.top_k_tanimoto_scores_file_name).exists() - assert (tmp_path / MS2QueryLibrary.reference_metadata_file_name).exists() + ReferenceLibrary.create_from_spectra(lib_spectra, ms2deepscore_model_file, tmp_path) + assert (tmp_path / ReferenceLibrary.embedding_file_name).exists() + assert (tmp_path / ReferenceLibrary.top_k_tanimoto_scores_file_name).exists() + assert (tmp_path / ReferenceLibrary.reference_metadata_file_name).exists() def test_create_and_use_library(tmp_path): lib_spectra = create_test_spectra(nr_of_inchikeys=10, number_of_spectra_per_inchikey=3) ms2deepscore_model_file = os.path.join(TEST_RESOURCES_PATH, "ms2deepscore_testmodel_v1.pt") - ms2query_library = MS2QueryLibrary.create_from_spectra(lib_spectra, ms2deepscore_model_file, tmp_path) + ms2query_library = ReferenceLibrary.create_from_spectra(lib_spectra, ms2deepscore_model_file, tmp_path) test_spectra = create_test_spectra(1, nr_of_inchikeys=3) results = ms2query_library.run_ms2query(test_spectra) - ms2query_library_2 = MS2QueryLibrary.load_from_files( + ms2query_library_2 = ReferenceLibrary.load_from_files( ms2deepscore_model_file, - tmp_path / MS2QueryLibrary.embedding_file_name, - tmp_path / MS2QueryLibrary.top_k_tanimoto_scores_file_name, - tmp_path / MS2QueryLibrary.reference_metadata_file_name, + tmp_path / ReferenceLibrary.embedding_file_name, + tmp_path / ReferenceLibrary.top_k_tanimoto_scores_file_name, + tmp_path / ReferenceLibrary.reference_metadata_file_name, ) results_2 = ms2query_library_2.run_ms2query(test_spectra) From e405b0467a73042580d1252d28e00b542d65d484 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 13:33:01 +0100 Subject: [PATCH 11/25] Rename benchmarking to ms2query_development --- .../AnnotatedSpectrumSet.py | 2 +- .../{benchmarking => ms2query_development}/Embeddings.py | 0 .../Fingerprints.py | 2 +- .../MS2DeepScoresForTopInChikeys.py | 8 ++++---- .../ReferenceLibrary.py | 8 ++++---- .../TopKTanimotoScores.py | 2 +- .../{benchmarking => ms2query_development}/__init__.py | 0 .../predict_top_k_ms2deepscore.py | 4 ++-- .../reference_methods/EvaluateExactMatchSearch.py | 2 +- .../reference_methods/__init__.py | 0 .../reference_methods/predict_best_possible_match.py | 4 ++-- .../reference_methods/readme.md | 0 12 files changed, 16 insertions(+), 16 deletions(-) rename ms2query/{benchmarking => ms2query_development}/AnnotatedSpectrumSet.py (99%) rename ms2query/{benchmarking => ms2query_development}/Embeddings.py (100%) rename ms2query/{benchmarking => ms2query_development}/Fingerprints.py (98%) rename ms2query/{benchmarking => ms2query_development}/MS2DeepScoresForTopInChikeys.py (95%) rename ms2query/{benchmarking => ms2query_development}/ReferenceLibrary.py (94%) rename ms2query/{benchmarking => ms2query_development}/TopKTanimotoScores.py (98%) rename ms2query/{benchmarking => ms2query_development}/__init__.py (100%) rename ms2query/{benchmarking => ms2query_development}/predict_top_k_ms2deepscore.py (96%) rename ms2query/{benchmarking => ms2query_development}/reference_methods/EvaluateExactMatchSearch.py (98%) rename ms2query/{benchmarking => ms2query_development}/reference_methods/__init__.py (100%) rename ms2query/{benchmarking => ms2query_development}/reference_methods/predict_best_possible_match.py (93%) rename ms2query/{benchmarking => ms2query_development}/reference_methods/readme.md (100%) diff --git a/ms2query/benchmarking/AnnotatedSpectrumSet.py b/ms2query/ms2query_development/AnnotatedSpectrumSet.py similarity index 99% rename from ms2query/benchmarking/AnnotatedSpectrumSet.py rename to ms2query/ms2query_development/AnnotatedSpectrumSet.py index 93ff130..bda0601 100644 --- a/ms2query/benchmarking/AnnotatedSpectrumSet.py +++ b/ms2query/ms2query_development/AnnotatedSpectrumSet.py @@ -6,7 +6,7 @@ from matchms.importing import load_spectra from ms2deepscore.models import SiameseSpectralModel from tqdm import tqdm -from ms2query.benchmarking.Embeddings import Embeddings +from ms2query.ms2query_development.Embeddings import Embeddings class AnnotatedSpectrumSet: diff --git a/ms2query/benchmarking/Embeddings.py b/ms2query/ms2query_development/Embeddings.py similarity index 100% rename from ms2query/benchmarking/Embeddings.py rename to ms2query/ms2query_development/Embeddings.py diff --git a/ms2query/benchmarking/Fingerprints.py b/ms2query/ms2query_development/Fingerprints.py similarity index 98% rename from ms2query/benchmarking/Fingerprints.py rename to ms2query/ms2query_development/Fingerprints.py index 4d7a9c1..eb90bc3 100644 --- a/ms2query/benchmarking/Fingerprints.py +++ b/ms2query/ms2query_development/Fingerprints.py @@ -5,7 +5,7 @@ from matchms.filtering.metadata_processing.add_fingerprint import _derive_fingerprint_from_inchi from numpy.typing import NDArray from tqdm import tqdm -from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet from ms2query.metrics import generalized_tanimoto_similarity_matrix diff --git a/ms2query/benchmarking/MS2DeepScoresForTopInChikeys.py b/ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py similarity index 95% rename from ms2query/benchmarking/MS2DeepScoresForTopInChikeys.py rename to ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py index d2519c7..7e20d27 100644 --- a/ms2query/benchmarking/MS2DeepScoresForTopInChikeys.py +++ b/ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py @@ -1,12 +1,12 @@ import numpy as np from ms2deepscore.vector_operations import cosine_similarity_matrix from tqdm import tqdm -from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.benchmarking.Fingerprints import Fingerprints -from ms2query.benchmarking.predict_top_k_ms2deepscore import ( +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.ms2query_development.Fingerprints import Fingerprints +from ms2query.ms2query_development.predict_top_k_ms2deepscore import ( select_inchikeys_with_highest_ms2deepscore, ) -from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores +from ms2query.ms2query_development.TopKTanimotoScores import TopKTanimotoScores def calculate_MS2DeepScoresForTopKInChikeys_from_spectra( diff --git a/ms2query/benchmarking/ReferenceLibrary.py b/ms2query/ms2query_development/ReferenceLibrary.py similarity index 94% rename from ms2query/benchmarking/ReferenceLibrary.py rename to ms2query/ms2query_development/ReferenceLibrary.py index 35ad4c7..37d2310 100644 --- a/ms2query/benchmarking/ReferenceLibrary.py +++ b/ms2query/ms2query_development/ReferenceLibrary.py @@ -8,10 +8,10 @@ from ms2deepscore.models import SiameseSpectralModel, load_model from ms2deepscore.vector_operations import cosine_similarity_matrix from tqdm import tqdm -from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.benchmarking.Embeddings import Embeddings, _to_json_serializable -from ms2query.benchmarking.Fingerprints import Fingerprints -from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.ms2query_development.Embeddings import Embeddings, _to_json_serializable +from ms2query.ms2query_development.Fingerprints import Fingerprints +from ms2query.ms2query_development.TopKTanimotoScores import TopKTanimotoScores class ReferenceLibrary: diff --git a/ms2query/benchmarking/TopKTanimotoScores.py b/ms2query/ms2query_development/TopKTanimotoScores.py similarity index 98% rename from ms2query/benchmarking/TopKTanimotoScores.py rename to ms2query/ms2query_development/TopKTanimotoScores.py index cd57271..d80462b 100644 --- a/ms2query/benchmarking/TopKTanimotoScores.py +++ b/ms2query/ms2query_development/TopKTanimotoScores.py @@ -1,7 +1,7 @@ from pathlib import Path import numpy as np import pandas as pd -from ms2query.benchmarking.Fingerprints import Fingerprints +from ms2query.ms2query_development.Fingerprints import Fingerprints from ms2query.metrics import generalized_tanimoto_similarity_matrix diff --git a/ms2query/benchmarking/__init__.py b/ms2query/ms2query_development/__init__.py similarity index 100% rename from ms2query/benchmarking/__init__.py rename to ms2query/ms2query_development/__init__.py diff --git a/ms2query/benchmarking/predict_top_k_ms2deepscore.py b/ms2query/ms2query_development/predict_top_k_ms2deepscore.py similarity index 96% rename from ms2query/benchmarking/predict_top_k_ms2deepscore.py rename to ms2query/ms2query_development/predict_top_k_ms2deepscore.py index 1da3b2e..149807c 100644 --- a/ms2query/benchmarking/predict_top_k_ms2deepscore.py +++ b/ms2query/ms2query_development/predict_top_k_ms2deepscore.py @@ -2,8 +2,8 @@ import numpy as np from ms2deepscore.vector_operations import cosine_similarity_matrix from tqdm import tqdm -from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.benchmarking.Embeddings import Embeddings +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.ms2query_development.Embeddings import Embeddings def predict_top_k_ms2deepscores( diff --git a/ms2query/benchmarking/reference_methods/EvaluateExactMatchSearch.py b/ms2query/ms2query_development/reference_methods/EvaluateExactMatchSearch.py similarity index 98% rename from ms2query/benchmarking/reference_methods/EvaluateExactMatchSearch.py rename to ms2query/ms2query_development/reference_methods/EvaluateExactMatchSearch.py index eced66b..a57368e 100644 --- a/ms2query/benchmarking/reference_methods/EvaluateExactMatchSearch.py +++ b/ms2query/ms2query_development/reference_methods/EvaluateExactMatchSearch.py @@ -1,7 +1,7 @@ import random from typing import Callable, List, Tuple from tqdm import tqdm -from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet class EvaluateExactMatchSearchAcrossIonmodes: diff --git a/ms2query/benchmarking/reference_methods/__init__.py b/ms2query/ms2query_development/reference_methods/__init__.py similarity index 100% rename from ms2query/benchmarking/reference_methods/__init__.py rename to ms2query/ms2query_development/reference_methods/__init__.py diff --git a/ms2query/benchmarking/reference_methods/predict_best_possible_match.py b/ms2query/ms2query_development/reference_methods/predict_best_possible_match.py similarity index 93% rename from ms2query/benchmarking/reference_methods/predict_best_possible_match.py rename to ms2query/ms2query_development/reference_methods/predict_best_possible_match.py index 1b82e04..e27d7ec 100644 --- a/ms2query/benchmarking/reference_methods/predict_best_possible_match.py +++ b/ms2query/ms2query_development/reference_methods/predict_best_possible_match.py @@ -1,7 +1,7 @@ from typing import Dict from matchms.similarity.vector_similarity_functions import jaccard_similarity_matrix -from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.benchmarking.Fingerprints import Fingerprints +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.ms2query_development.Fingerprints import Fingerprints def predict_best_possible_match( diff --git a/ms2query/benchmarking/reference_methods/readme.md b/ms2query/ms2query_development/reference_methods/readme.md similarity index 100% rename from ms2query/benchmarking/reference_methods/readme.md rename to ms2query/ms2query_development/reference_methods/readme.md From aebacf45359737ff2347c4ce77c09646e128cb09 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 13:35:20 +0100 Subject: [PATCH 12/25] Apply change of folder name in tests an notebook --- .../develop_and_compare_ms2query_2.ipynb | 30 +++++++++---------- tests/helper_functions.py | 4 +-- tests/test_benchmarking/test_Fingerprints.py | 4 +-- .../test_MS2DeepScoresForTopInChikeys.py | 4 +-- .../test_benchmarking/test_SpectrumDataSet.py | 2 +- tests/test_benchmarking/test_embeddings.py | 2 +- tests/test_benchmarking/test_methods.py | 4 +-- .../test_predict_top_ms2deepscores.py | 4 +-- .../test_top_k_tanimoto_scores.py | 2 +- tests/test_run_ms2query.py | 8 ++--- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb b/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb index 4b244b2..b384c6f 100644 --- a/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb +++ b/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb @@ -252,7 +252,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "6fecef86-84af-40e9-bf48-d81d62f8bcf4", "metadata": {}, "outputs": [ @@ -270,7 +270,7 @@ } ], "source": [ - "from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet\n", + "from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet\n", "\n", "pos_test_spectra = AnnotatedSpectrumSet.create_spectrum_set(pos_test_spectra)\n", "pos_train_spectra = AnnotatedSpectrumSet.create_spectrum_set(pos_train_spectra)\n", @@ -279,7 +279,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "92a0e90c-d435-4a0e-b9ba-bc7ab2139ca6", "metadata": {}, "outputs": [ @@ -294,7 +294,7 @@ } ], "source": [ - "from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet\n", + "from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet\n", "\n", "neg_val_spectra = AnnotatedSpectrumSet.create_spectrum_set(neg_val_spectra)\n", "neg_test_spectra = AnnotatedSpectrumSet.create_spectrum_set(neg_test_spectra)\n", @@ -428,7 +428,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "3453717f-b50d-4623-bd87-b1287c0fe916", "metadata": {}, "outputs": [ @@ -444,7 +444,7 @@ } ], "source": [ - "from ms2query.benchmarking.Fingerprints import Fingerprints\n", + "from ms2query.ms2query_development.Fingerprints import Fingerprints\n", "val_fingerprints = Fingerprints.from_spectrum_set(pos_val_spectra, \"daylight\", 4096)\n", "train_fingerprints = Fingerprints.from_spectrum_set(pos_train_spectra, \"daylight\", 4096)\n", "val_and_train_fingerprints = Fingerprints.combine_fingerprints(val_fingerprints, train_fingerprints)" @@ -461,12 +461,12 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": null, "id": "cc395ecc-9473-4c9d-979f-ff8c087293d5", "metadata": {}, "outputs": [], "source": [ - "from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores\n", + "from ms2query.ms2query_development.TopKTanimotoScores import TopKTanimotoScores\n", "top_k_tanimoto_scores = TopKTanimotoScores.calculate_from_fingerprints(\n", " train_fingerprints,\n", " train_fingerprints, k=100,)" @@ -488,7 +488,7 @@ "metadata": {}, "outputs": [], "source": [ - "from ms2query.benchmarking.predict_top_k_ms2deepscore import select_inchikeys_with_highest_ms2deepscore\n", + "from ms2query.ms2query_development.predict_top_k_ms2deepscore import select_inchikeys_with_highest_ms2deepscore\n", "\n", "inchikeys_with_highest_ms2deepscores = select_inchikeys_with_highest_ms2deepscore(\n", " pos_val_spectra, pos_train_spectra, nr_of_inchikeys_to_select=1, batch_size=1000,)\n" @@ -509,7 +509,7 @@ } ], "source": [ - "from ms2query.benchmarking.MS2DeepScoresForTopInChikeys import calculate_MS2DeepScoresForTopKInChikeys\n", + "from ms2query.ms2query_development.MS2DeepScoresForTopInChikeys import calculate_MS2DeepScoresForTopKInChikeys\n", "close_tanimoto_scores = calculate_MS2DeepScoresForTopKInChikeys(pos_train_spectra, pos_val_spectra, top_k_tanimoto_scores, inchikeys_with_highest_ms2deepscores)" ] }, @@ -2630,10 +2630,10 @@ } ], "source": [ - "from ms2query.benchmarking.Fingerprints import Fingerprints\n", - "from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores\n", - "from ms2query.benchmarking.predict_top_k_ms2deepscore import select_inchikeys_with_highest_ms2deepscore\n", - "from ms2query.benchmarking.MS2DeepScoresForTopInChikeys import calculate_MS2DeepScoresForTopKInChikeys\n", + "from ms2query.ms2query_development.Fingerprints import Fingerprints\n", + "from ms2query.ms2query_development.TopKTanimotoScores import TopKTanimotoScores\n", + "from ms2query.ms2query_development.predict_top_k_ms2deepscore import select_inchikeys_with_highest_ms2deepscore\n", + "from ms2query.ms2query_development.MS2DeepScoresForTopInChikeys import calculate_MS2DeepScoresForTopKInChikeys\n", "\n", "neg_val_fingerprints = Fingerprints.from_spectrum_set(neg_val_spectra, \"daylight\", 4096)\n", "neg_train_fingerprints = Fingerprints.from_spectrum_set(neg_train_spectra, \"daylight\", 4096)\n", @@ -2714,7 +2714,7 @@ "metadata": {}, "outputs": [], "source": [ - "from ms2query.benchmarking.predict_top_k_ms2deepscore import predict_top_k_ms2deepscores\n", + "from ms2query.ms2query_development.predict_top_k_ms2deepscore import predict_top_k_ms2deepscores\n", "\n", "indexes, ms2deepscores = predict_top_k_ms2deepscores(neg_train_spectra.embeddings, neg_val_spectra.embeddings)\n", "neg_predicted_inchikeys_ms2deepscore = [neg_train_spectra.spectra[int(index)].get(\"inchikey\")[:14] for index in indexes]\n", diff --git a/tests/helper_functions.py b/tests/helper_functions.py index 875ae13..5c234b4 100644 --- a/tests/helper_functions.py +++ b/tests/helper_functions.py @@ -4,8 +4,8 @@ import numpy as np from matchms.Spectrum import Spectrum from ms2deepscore.models import load_model -from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.benchmarking.Fingerprints import Fingerprints +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.ms2query_development.Fingerprints import Fingerprints TEST_RESOURCES_PATH = Path(__file__).parent / "test_data" diff --git a/tests/test_benchmarking/test_Fingerprints.py b/tests/test_benchmarking/test_Fingerprints.py index 6ab8083..6bffc74 100644 --- a/tests/test_benchmarking/test_Fingerprints.py +++ b/tests/test_benchmarking/test_Fingerprints.py @@ -1,8 +1,8 @@ import numpy as np import pytest from matchms.filtering.metadata_processing.add_fingerprint import _derive_fingerprint_from_inchi -from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.benchmarking.Fingerprints import Fingerprints, get_similarity_matrix +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.ms2query_development.Fingerprints import Fingerprints, get_similarity_matrix from tests.helper_functions import create_test_spectra, get_inchikey_inchi_pairs diff --git a/tests/test_benchmarking/test_MS2DeepScoresForTopInChikeys.py b/tests/test_benchmarking/test_MS2DeepScoresForTopInChikeys.py index f0ed9df..7c1a7a4 100644 --- a/tests/test_benchmarking/test_MS2DeepScoresForTopInChikeys.py +++ b/tests/test_benchmarking/test_MS2DeepScoresForTopInChikeys.py @@ -1,5 +1,5 @@ -from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.benchmarking.MS2DeepScoresForTopInChikeys import ( +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.ms2query_development.MS2DeepScoresForTopInChikeys import ( calculate_MS2DeepScoresForTopKInChikeys_from_spectra, ) from tests.helper_functions import create_test_spectra, ms2deepscore_model diff --git a/tests/test_benchmarking/test_SpectrumDataSet.py b/tests/test_benchmarking/test_SpectrumDataSet.py index 3e252eb..94baa36 100644 --- a/tests/test_benchmarking/test_SpectrumDataSet.py +++ b/tests/test_benchmarking/test_SpectrumDataSet.py @@ -1,6 +1,6 @@ import os import pytest -from ms2query.benchmarking.AnnotatedSpectrumSet import ( +from ms2query.ms2query_development.AnnotatedSpectrumSet import ( AnnotatedSpectrumSet, ) from tests.helper_functions import create_test_spectra, ms2deepscore_model diff --git a/tests/test_benchmarking/test_embeddings.py b/tests/test_benchmarking/test_embeddings.py index 86fbbbd..5ad42c9 100644 --- a/tests/test_benchmarking/test_embeddings.py +++ b/tests/test_benchmarking/test_embeddings.py @@ -1,6 +1,6 @@ import os import pytest -from ms2query.benchmarking.Embeddings import Embeddings, calculate_ms2deepscore_df +from ms2query.ms2query_development.Embeddings import Embeddings, calculate_ms2deepscore_df from tests.helper_functions import create_test_spectra, get_library_and_test_spectra_not_identical, ms2deepscore_model diff --git a/tests/test_benchmarking/test_methods.py b/tests/test_benchmarking/test_methods.py index b584df4..240c11b 100644 --- a/tests/test_benchmarking/test_methods.py +++ b/tests/test_benchmarking/test_methods.py @@ -1,6 +1,6 @@ import numpy as np -from ms2query.benchmarking.Fingerprints import Fingerprints -from ms2query.benchmarking.reference_methods.predict_best_possible_match import predict_best_possible_match +from ms2query.ms2query_development.Fingerprints import Fingerprints +from ms2query.ms2query_development.reference_methods.predict_best_possible_match import predict_best_possible_match from tests.helper_functions import ( get_library_and_test_spectra_not_identical, ) diff --git a/tests/test_benchmarking/test_predict_top_ms2deepscores.py b/tests/test_benchmarking/test_predict_top_ms2deepscores.py index e830017..6fe6b49 100644 --- a/tests/test_benchmarking/test_predict_top_ms2deepscores.py +++ b/tests/test_benchmarking/test_predict_top_ms2deepscores.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.benchmarking.predict_top_k_ms2deepscore import ( +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.ms2query_development.predict_top_k_ms2deepscore import ( predict_top_k_ms2deepscores, select_inchikeys_with_highest_ms2deepscore, ) diff --git a/tests/test_benchmarking/test_top_k_tanimoto_scores.py b/tests/test_benchmarking/test_top_k_tanimoto_scores.py index bff7ecc..e15bb15 100644 --- a/tests/test_benchmarking/test_top_k_tanimoto_scores.py +++ b/tests/test_benchmarking/test_top_k_tanimoto_scores.py @@ -1,7 +1,7 @@ import numpy as np import pandas as pd import pytest -from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores +from ms2query.ms2query_development.TopKTanimotoScores import TopKTanimotoScores from tests.helper_functions import make_test_fingerprints diff --git a/tests/test_run_ms2query.py b/tests/test_run_ms2query.py index 63714cc..c28bccc 100644 --- a/tests/test_run_ms2query.py +++ b/tests/test_run_ms2query.py @@ -1,9 +1,9 @@ import os import pandas as pd -from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.benchmarking.Fingerprints import Fingerprints -from ms2query.benchmarking.TopKTanimotoScores import TopKTanimotoScores -from ms2query.benchmarking.ReferenceLibrary import ( +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet +from ms2query.ms2query_development.Fingerprints import Fingerprints +from ms2query.ms2query_development.TopKTanimotoScores import TopKTanimotoScores +from ms2query.ms2query_development.ReferenceLibrary import ( ReferenceLibrary, extract_metadata_from_library, run_ms2query_from_files, From 4ba50fd7b0e4f4c111ec2582daa32e4277355bdb Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 15:02:17 +0100 Subject: [PATCH 13/25] Add note that it is only needed for benchmarking --- ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py b/ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py index 7e20d27..17233b9 100644 --- a/ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py +++ b/ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py @@ -77,6 +77,8 @@ def calculate_MS2DeepScoresForTopKInChikeys( class MS2DeepScoresForTopKInChikeys: """Stores the MS2DeepScores and Tanimoto scores for the top k closest lib spectra + This is only needed for the benchmarking and development (in the notebooks) and is not used for running the final verison of MS2Query + This allows for quick testing of different reranking strategies. E.g. get_mean is similar to the original MS2Query, but it can also be used to make matrixes with both MS2DeepScore and tanimoto scores to train small reranking models. From 8e649938802f55689803ec05d10a05b909784576 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 15:07:04 +0100 Subject: [PATCH 14/25] Change test_benchmarking ot test_ms2query_development --- .../test_Fingerprints.py | 0 .../test_MS2DeepScoresForTopInChikeys.py | 0 .../test_SpectrumDataSet.py | 0 .../test_embeddings.py | 0 .../test_methods.py | 0 .../test_predict_top_ms2deepscores.py | 0 tests/{ => test_ms2query_development}/test_run_ms2query.py | 3 +-- .../test_top_k_tanimoto_scores.py | 0 8 files changed, 1 insertion(+), 2 deletions(-) rename tests/{test_benchmarking => test_ms2query_development}/test_Fingerprints.py (100%) rename tests/{test_benchmarking => test_ms2query_development}/test_MS2DeepScoresForTopInChikeys.py (100%) rename tests/{test_benchmarking => test_ms2query_development}/test_SpectrumDataSet.py (100%) rename tests/{test_benchmarking => test_ms2query_development}/test_embeddings.py (100%) rename tests/{test_benchmarking => test_ms2query_development}/test_methods.py (100%) rename tests/{test_benchmarking => test_ms2query_development}/test_predict_top_ms2deepscores.py (100%) rename tests/{ => test_ms2query_development}/test_run_ms2query.py (96%) rename tests/{test_benchmarking => test_ms2query_development}/test_top_k_tanimoto_scores.py (100%) diff --git a/tests/test_benchmarking/test_Fingerprints.py b/tests/test_ms2query_development/test_Fingerprints.py similarity index 100% rename from tests/test_benchmarking/test_Fingerprints.py rename to tests/test_ms2query_development/test_Fingerprints.py diff --git a/tests/test_benchmarking/test_MS2DeepScoresForTopInChikeys.py b/tests/test_ms2query_development/test_MS2DeepScoresForTopInChikeys.py similarity index 100% rename from tests/test_benchmarking/test_MS2DeepScoresForTopInChikeys.py rename to tests/test_ms2query_development/test_MS2DeepScoresForTopInChikeys.py diff --git a/tests/test_benchmarking/test_SpectrumDataSet.py b/tests/test_ms2query_development/test_SpectrumDataSet.py similarity index 100% rename from tests/test_benchmarking/test_SpectrumDataSet.py rename to tests/test_ms2query_development/test_SpectrumDataSet.py diff --git a/tests/test_benchmarking/test_embeddings.py b/tests/test_ms2query_development/test_embeddings.py similarity index 100% rename from tests/test_benchmarking/test_embeddings.py rename to tests/test_ms2query_development/test_embeddings.py diff --git a/tests/test_benchmarking/test_methods.py b/tests/test_ms2query_development/test_methods.py similarity index 100% rename from tests/test_benchmarking/test_methods.py rename to tests/test_ms2query_development/test_methods.py diff --git a/tests/test_benchmarking/test_predict_top_ms2deepscores.py b/tests/test_ms2query_development/test_predict_top_ms2deepscores.py similarity index 100% rename from tests/test_benchmarking/test_predict_top_ms2deepscores.py rename to tests/test_ms2query_development/test_predict_top_ms2deepscores.py diff --git a/tests/test_run_ms2query.py b/tests/test_ms2query_development/test_run_ms2query.py similarity index 96% rename from tests/test_run_ms2query.py rename to tests/test_ms2query_development/test_run_ms2query.py index c28bccc..77b5bf9 100644 --- a/tests/test_run_ms2query.py +++ b/tests/test_ms2query_development/test_run_ms2query.py @@ -2,12 +2,11 @@ import pandas as pd from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet from ms2query.ms2query_development.Fingerprints import Fingerprints -from ms2query.ms2query_development.TopKTanimotoScores import TopKTanimotoScores from ms2query.ms2query_development.ReferenceLibrary import ( ReferenceLibrary, extract_metadata_from_library, - run_ms2query_from_files, ) +from ms2query.ms2query_development.TopKTanimotoScores import TopKTanimotoScores from tests.helper_functions import TEST_RESOURCES_PATH, create_test_spectra, ms2deepscore_model diff --git a/tests/test_benchmarking/test_top_k_tanimoto_scores.py b/tests/test_ms2query_development/test_top_k_tanimoto_scores.py similarity index 100% rename from tests/test_benchmarking/test_top_k_tanimoto_scores.py rename to tests/test_ms2query_development/test_top_k_tanimoto_scores.py From 7286c8c7c447bc1d2e4a73c5c524d1ad001f3392 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 15:07:18 +0100 Subject: [PATCH 15/25] Sort import statements --- ms2query/ms2query_development/Fingerprints.py | 2 +- ms2query/ms2query_development/TopKTanimotoScores.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ms2query/ms2query_development/Fingerprints.py b/ms2query/ms2query_development/Fingerprints.py index eb90bc3..4e20cce 100644 --- a/ms2query/ms2query_development/Fingerprints.py +++ b/ms2query/ms2query_development/Fingerprints.py @@ -5,8 +5,8 @@ from matchms.filtering.metadata_processing.add_fingerprint import _derive_fingerprint_from_inchi from numpy.typing import NDArray from tqdm import tqdm -from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet from ms2query.metrics import generalized_tanimoto_similarity_matrix +from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet class Fingerprints: diff --git a/ms2query/ms2query_development/TopKTanimotoScores.py b/ms2query/ms2query_development/TopKTanimotoScores.py index d80462b..c930ecc 100644 --- a/ms2query/ms2query_development/TopKTanimotoScores.py +++ b/ms2query/ms2query_development/TopKTanimotoScores.py @@ -1,8 +1,8 @@ from pathlib import Path import numpy as np import pandas as pd -from ms2query.ms2query_development.Fingerprints import Fingerprints from ms2query.metrics import generalized_tanimoto_similarity_matrix +from ms2query.ms2query_development.Fingerprints import Fingerprints class TopKTanimotoScores: From 82ee3bb5052685ac53b2fd783fa26fce2109b17a Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 15:13:51 +0100 Subject: [PATCH 16/25] linting --- ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py | 3 ++- ms2query/ms2query_development/ReferenceLibrary.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py b/ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py index 17233b9..90c8692 100644 --- a/ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py +++ b/ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py @@ -77,7 +77,8 @@ def calculate_MS2DeepScoresForTopKInChikeys( class MS2DeepScoresForTopKInChikeys: """Stores the MS2DeepScores and Tanimoto scores for the top k closest lib spectra - This is only needed for the benchmarking and development (in the notebooks) and is not used for running the final verison of MS2Query + This is only needed for the benchmarking and development (in the notebooks) + and is not used for running the final verison of MS2Query This allows for quick testing of different reranking strategies. E.g. get_mean is similar to the original MS2Query, but it can also be used to make matrixes with both MS2DeepScore and tanimoto scores to train small reranking models. diff --git a/ms2query/ms2query_development/ReferenceLibrary.py b/ms2query/ms2query_development/ReferenceLibrary.py index 37d2310..4ba5e4a 100644 --- a/ms2query/ms2query_development/ReferenceLibrary.py +++ b/ms2query/ms2query_development/ReferenceLibrary.py @@ -47,7 +47,7 @@ def __init__( # Check that the loaded files match if _to_json_serializable(ms2deepscore_model.model_settings.get_dict()) != reference_embeddings.model_settings: raise ValueError( - "The settings of the ms2deepscore model does not match the model used for creating the library embeddings" + "The settings of the ms2deepscore model do not match the model used for creating the library embeddings" ) if list(self.reference_metadata["spectrum_hashes"]) != [ str(spectrum_hash) for spectrum_hash in reference_embeddings.index_to_spectrum_hash From d55cff1faf9543d4532a55249e112e53e4ee4ff1 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 16:38:48 +0100 Subject: [PATCH 17/25] Add a first run MS2Query tutorial --- ms2query/notebooks/tutorial.ipynb | 196 ++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 ms2query/notebooks/tutorial.ipynb diff --git a/ms2query/notebooks/tutorial.ipynb b/ms2query/notebooks/tutorial.ipynb new file mode 100644 index 0000000..411561f --- /dev/null +++ b/ms2query/notebooks/tutorial.ipynb @@ -0,0 +1,196 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "70e60490", + "metadata": {}, + "source": [ + "# Tutorial for running MS2Query and creating your own libraries" + ] + }, + { + "cell_type": "markdown", + "id": "4d564033", + "metadata": {}, + "source": [ + "# optional: download a matchms cleaned library\n", + "The code below downloads an already matchms cleaned library and an MS2DeepScore model. You can also use your own library, but make sure you know what you are doing and clean the library first. If you just have a few reference spectra, it is probably best to combine your spectra with the reference spectra below to make sure the MS2Query search works properly. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7df418e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The file ./zenodo_files\\data_split_inchikeys.json already exists, the file won't be downloaded\n", + "The file ./zenodo_files\\merged_and_cleaned_libraries_1.mgf already exists, the file won't be downloaded\n", + "The file ./zenodo_files\\ms2deepscore_model.pt already exists, the file won't be downloaded\n" + ] + } + ], + "source": [ + "import requests\n", + "import os\n", + "from tqdm import tqdm\n", + "\n", + "def download_file(link, file_name):\n", + " response = requests.get(link, stream=True)\n", + " if os.path.exists(file_name):\n", + " print(f\"The file {file_name} already exists, the file won't be downloaded\")\n", + " return\n", + " total_size = int(response.headers.get('content-length', 0))\n", + "\n", + " with open(file_name, \"wb\") as f, tqdm(desc=\"Downloading file\", total=total_size, unit='B', unit_scale=True, unit_divisor=1024,) as bar:\n", + " for chunk in response.iter_content(chunk_size=1024):\n", + " if chunk:\n", + " f.write(chunk)\n", + " bar.update(len(chunk)) # Update progress bar by the chunk size\n", + "folder_to_store_zenodo_files = \"./zenodo_files\"\n", + "os.makedirs(folder_to_store_zenodo_files, exist_ok=True)\n", + "\n", + "download_file(\"https://zenodo.org/records/16882111/files/merged_and_cleaned_libraries_1.mgf?download=1\", \n", + " os.path.join(folder_to_store_zenodo_files, \"merged_and_cleaned_libraries_1.mgf\"))\n", + "download_file(\"https://zenodo.org/records/17826815/files/ms2deepscore_model.pt?download=1\", \n", + " os.path.join(folder_to_store_zenodo_files, \"ms2deepscore_model.pt\"))" + ] + }, + { + "cell_type": "markdown", + "id": "fa99d2dd", + "metadata": {}, + "source": [ + "# Specify file location \n", + "Replace with your file names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6db579a9", + "metadata": {}, + "outputs": [], + "source": [ + "library_spectra_file = os.path.join(folder_to_store_zenodo_files, \"merged_and_cleaned_libraries_1.mgf\")\n", + "ms2deepscore_model_file_name = os.path.join(folder_to_store_zenodo_files, \"ms2deepscore_model.pt\")\n", + "query_spectrum_file = \"replace_with_your_lib_spectra.mgf\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f521c1c4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "1017531it [09:51, 1720.67it/s]\n" + ] + } + ], + "source": [ + "from matchms.importing import load_from_mgf\n", + "from tqdm import tqdm\n", + "\n", + "library_spectra = list(tqdm(load_from_mgf(library_spectra_file)))\n", + "query_spectra = list(tqdm(load_from_mgf(query_spectrum_file)))" + ] + }, + { + "cell_type": "markdown", + "id": "64534dd6", + "metadata": {}, + "source": [ + "# Create the reference library files\n", + "The code below will precompute everything needed to run MS2Query. It will save this in the same folder as your ms2deepscore model. \n", + "The files created are \"embeddings.npz\", \"top_k_tanimoto_scores.parquet\", \"library_metadata.parquet\". " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "669a489a", + "metadata": {}, + "outputs": [], + "source": [ + "from ms2query.ms2query_development.ReferenceLibrary import ReferenceLibrary\n", + "reference_library = ReferenceLibrary.create_from_spectra(library_spectra, ms2deepscore_model_file_name)" + ] + }, + { + "cell_type": "markdown", + "id": "dfd0a633", + "metadata": {}, + "source": [ + "# Run MS2Query\n", + "The code above only has to be run once after that you can load the library faster from the saved files. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6cf44e2", + "metadata": {}, + "outputs": [], + "source": [ + "# no need to run if you just created the libary above\n", + "reference_library = ReferenceLibrary.load_from_directory(folder_to_store_zenodo_files)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "566b2b0d", + "metadata": {}, + "outputs": [], + "source": [ + "results = reference_library.run_ms2query(query_spectra)" + ] + }, + { + "cell_type": "markdown", + "id": "98ce9660", + "metadata": {}, + "source": [ + "print(results)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7e3b533", + "metadata": {}, + "outputs": [], + "source": [ + "results.to_csv(\"ms2query_results.csv\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ms2query_2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From d1ece964027e9b6e5c495421fa3bf6488d78f040 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Mon, 23 Mar 2026 17:04:57 +0100 Subject: [PATCH 18/25] Point to tutorial notebook in readme --- README.md | 27 +++++---------------------- ms2query/readme.md | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 22 deletions(-) create mode 100644 ms2query/readme.md diff --git a/README.md b/README.md index 4fe0e73..5c1ec45 100644 --- a/README.md +++ b/README.md @@ -2,28 +2,11 @@ # MS2Query 2.0 -more to come... - -## Basic workflow (so far): - -### Library generation -```python -from ms2query.create_new_library import create_new_library - -ms2query_lib = create_new_library( - spectra_files=["spectra.mgf"], - annotation_files=[], - output_folder="my_ms2query_folder/", - model_path="models/ms2deepscore.pt" -) -``` - -### Loading already generated library -```python -from ms2query.create_new_library import load_created_library - -lib = load_created_library("my_ms2query_folder/") -``` +A first basic implementation is out now, more to follow soon... +The new MS2Query appraoch has a higher accuracy and has a much simpler and faster underlying algorithm. We will hopefully soon share a first preprint as well, showing all the benchmarking. +The current runably version still requires to create the library files, which takes some time for the first run. +Soon this will be much easier and faster. We will add downloadable precomputed files, make MS2Query pip installable, add a database and allow faster MS2DeepScore searching. +The tutorial for the current prototype can be found in notebooks/tutorial. diff --git a/ms2query/readme.md b/ms2query/readme.md new file mode 100644 index 0000000..b2b1267 --- /dev/null +++ b/ms2query/readme.md @@ -0,0 +1,21 @@ +## Basic workflow (so far): +This is for creating the database. This is not yet fully functional, so for now please use the notebooks/tutorial.ipynb if you already want to try out the prototype. + +### Library generation +```python +from ms2query.create_new_library import create_new_library + +ms2query_lib = create_new_library( + spectra_files=["spectra.mgf"], + annotation_files=[], + output_folder="my_ms2query_folder/", + model_path="models/ms2deepscore.pt" +) +``` + +### Loading already generated library +```python +from ms2query.create_new_library import load_created_library + +lib = load_created_library("my_ms2query_folder/") +``` From 25508a6ea5e0bd03924e2efdb4433510403c65e1 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Tue, 24 Mar 2026 09:42:23 +0100 Subject: [PATCH 19/25] Move MS2DeepScoresForTopInChikeys since not used for core functionality --- .../{ => reference_methods}/MS2DeepScoresForTopInChikeys.py | 0 ms2query/ms2query_development/reference_methods/readme.md | 2 +- ms2query/notebooks/develop_and_compare_ms2query_2.ipynb | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename ms2query/ms2query_development/{ => reference_methods}/MS2DeepScoresForTopInChikeys.py (100%) diff --git a/ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py b/ms2query/ms2query_development/reference_methods/MS2DeepScoresForTopInChikeys.py similarity index 100% rename from ms2query/ms2query_development/MS2DeepScoresForTopInChikeys.py rename to ms2query/ms2query_development/reference_methods/MS2DeepScoresForTopInChikeys.py diff --git a/ms2query/ms2query_development/reference_methods/readme.md b/ms2query/ms2query_development/reference_methods/readme.md index 18eab2f..7d2efbe 100644 --- a/ms2query/ms2query_development/reference_methods/readme.md +++ b/ms2query/ms2query_development/reference_methods/readme.md @@ -1 +1 @@ -All files here are not important to core functionality and in fact not used, but might be nice to use for future testing. They are currently not yet used in the notebooks. Once they are used there they can probably be removed here. +All files here are not important to core functionality and in fact not used for running the final version of MS2Query. However MS2DeepSCoresForTopInChikeys is core to the benchmarking in the notebooks. predict_best_possible_match and EvaluateExactMatchSearch are not yet used for the benchmarking, but both should still be done, so this could be used for that. diff --git a/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb b/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb index b384c6f..b5e6786 100644 --- a/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb +++ b/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb @@ -509,7 +509,7 @@ } ], "source": [ - "from ms2query.ms2query_development.MS2DeepScoresForTopInChikeys import calculate_MS2DeepScoresForTopKInChikeys\n", + "from ms2query.ms2query_development.reference_methods.MS2DeepScoresForTopInChikeys import calculate_MS2DeepScoresForTopKInChikeys\n", "close_tanimoto_scores = calculate_MS2DeepScoresForTopKInChikeys(pos_train_spectra, pos_val_spectra, top_k_tanimoto_scores, inchikeys_with_highest_ms2deepscores)" ] }, @@ -2633,7 +2633,7 @@ "from ms2query.ms2query_development.Fingerprints import Fingerprints\n", "from ms2query.ms2query_development.TopKTanimotoScores import TopKTanimotoScores\n", "from ms2query.ms2query_development.predict_top_k_ms2deepscore import select_inchikeys_with_highest_ms2deepscore\n", - "from ms2query.ms2query_development.MS2DeepScoresForTopInChikeys import calculate_MS2DeepScoresForTopKInChikeys\n", + "from ms2query.ms2query_development.reference_methods.MS2DeepScoresForTopInChikeys import calculate_MS2DeepScoresForTopKInChikeys\n", "\n", "neg_val_fingerprints = Fingerprints.from_spectrum_set(neg_val_spectra, \"daylight\", 4096)\n", "neg_train_fingerprints = Fingerprints.from_spectrum_set(neg_train_spectra, \"daylight\", 4096)\n", From bd5a783c1b3427792ee0aa2deffd3fd630b5cc90 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Tue, 24 Mar 2026 09:44:06 +0100 Subject: [PATCH 20/25] Move predict_top_k_ms2deepscore as well --- .../{ => reference_methods}/predict_top_k_ms2deepscore.py | 0 ms2query/ms2query_development/reference_methods/readme.md | 2 +- ms2query/notebooks/develop_and_compare_ms2query_2.ipynb | 6 +++--- .../test_MS2DeepScoresForTopInChikeys.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename ms2query/ms2query_development/{ => reference_methods}/predict_top_k_ms2deepscore.py (100%) diff --git a/ms2query/ms2query_development/predict_top_k_ms2deepscore.py b/ms2query/ms2query_development/reference_methods/predict_top_k_ms2deepscore.py similarity index 100% rename from ms2query/ms2query_development/predict_top_k_ms2deepscore.py rename to ms2query/ms2query_development/reference_methods/predict_top_k_ms2deepscore.py diff --git a/ms2query/ms2query_development/reference_methods/readme.md b/ms2query/ms2query_development/reference_methods/readme.md index 7d2efbe..289ff6a 100644 --- a/ms2query/ms2query_development/reference_methods/readme.md +++ b/ms2query/ms2query_development/reference_methods/readme.md @@ -1 +1 @@ -All files here are not important to core functionality and in fact not used for running the final version of MS2Query. However MS2DeepSCoresForTopInChikeys is core to the benchmarking in the notebooks. predict_best_possible_match and EvaluateExactMatchSearch are not yet used for the benchmarking, but both should still be done, so this could be used for that. +All files here are not important to core functionality and in fact not used for running the final version of MS2Query. However MS2DeepSCoresForTopInChikeys and predict_top-k_ms2deepscore are core to the benchmarking and experimenting in the notebooks. predict_best_possible_match and EvaluateExactMatchSearch are not yet used for the benchmarking, but both should still be done, so this could be used for that. diff --git a/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb b/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb index b5e6786..80a3ba6 100644 --- a/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb +++ b/ms2query/notebooks/develop_and_compare_ms2query_2.ipynb @@ -488,7 +488,7 @@ "metadata": {}, "outputs": [], "source": [ - "from ms2query.ms2query_development.predict_top_k_ms2deepscore import select_inchikeys_with_highest_ms2deepscore\n", + "from ms2query.ms2query_development.reference_methods.predict_top_k_ms2deepscore import select_inchikeys_with_highest_ms2deepscore\n", "\n", "inchikeys_with_highest_ms2deepscores = select_inchikeys_with_highest_ms2deepscore(\n", " pos_val_spectra, pos_train_spectra, nr_of_inchikeys_to_select=1, batch_size=1000,)\n" @@ -2632,7 +2632,7 @@ "source": [ "from ms2query.ms2query_development.Fingerprints import Fingerprints\n", "from ms2query.ms2query_development.TopKTanimotoScores import TopKTanimotoScores\n", - "from ms2query.ms2query_development.predict_top_k_ms2deepscore import select_inchikeys_with_highest_ms2deepscore\n", + "from ms2query.ms2query_development.reference_methods.predict_top_k_ms2deepscore import select_inchikeys_with_highest_ms2deepscore\n", "from ms2query.ms2query_development.reference_methods.MS2DeepScoresForTopInChikeys import calculate_MS2DeepScoresForTopKInChikeys\n", "\n", "neg_val_fingerprints = Fingerprints.from_spectrum_set(neg_val_spectra, \"daylight\", 4096)\n", @@ -2714,7 +2714,7 @@ "metadata": {}, "outputs": [], "source": [ - "from ms2query.ms2query_development.predict_top_k_ms2deepscore import predict_top_k_ms2deepscores\n", + "from ms2query.ms2query_development.reference_methods.predict_top_k_ms2deepscore import predict_top_k_ms2deepscores\n", "\n", "indexes, ms2deepscores = predict_top_k_ms2deepscores(neg_train_spectra.embeddings, neg_val_spectra.embeddings)\n", "neg_predicted_inchikeys_ms2deepscore = [neg_train_spectra.spectra[int(index)].get(\"inchikey\")[:14] for index in indexes]\n", diff --git a/tests/test_ms2query_development/test_MS2DeepScoresForTopInChikeys.py b/tests/test_ms2query_development/test_MS2DeepScoresForTopInChikeys.py index 7c1a7a4..a065fed 100644 --- a/tests/test_ms2query_development/test_MS2DeepScoresForTopInChikeys.py +++ b/tests/test_ms2query_development/test_MS2DeepScoresForTopInChikeys.py @@ -1,5 +1,5 @@ from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.ms2query_development.MS2DeepScoresForTopInChikeys import ( +from ms2query.ms2query_development.reference_methods.MS2DeepScoresForTopInChikeys import ( calculate_MS2DeepScoresForTopKInChikeys_from_spectra, ) from tests.helper_functions import create_test_spectra, ms2deepscore_model From 449cfc47e173de8bad521574aedaa013796db86c Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Tue, 24 Mar 2026 09:46:16 +0100 Subject: [PATCH 21/25] Complete move of predict_top_k_ms2deepscores --- .../reference_methods/MS2DeepScoresForTopInChikeys.py | 2 +- .../test_ms2query_development/test_predict_top_ms2deepscores.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ms2query/ms2query_development/reference_methods/MS2DeepScoresForTopInChikeys.py b/ms2query/ms2query_development/reference_methods/MS2DeepScoresForTopInChikeys.py index 90c8692..883ab35 100644 --- a/ms2query/ms2query_development/reference_methods/MS2DeepScoresForTopInChikeys.py +++ b/ms2query/ms2query_development/reference_methods/MS2DeepScoresForTopInChikeys.py @@ -3,7 +3,7 @@ from tqdm import tqdm from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet from ms2query.ms2query_development.Fingerprints import Fingerprints -from ms2query.ms2query_development.predict_top_k_ms2deepscore import ( +from ms2query.ms2query_development.reference_methods.predict_top_k_ms2deepscore import ( select_inchikeys_with_highest_ms2deepscore, ) from ms2query.ms2query_development.TopKTanimotoScores import TopKTanimotoScores diff --git a/tests/test_ms2query_development/test_predict_top_ms2deepscores.py b/tests/test_ms2query_development/test_predict_top_ms2deepscores.py index 6fe6b49..7cd00ce 100644 --- a/tests/test_ms2query_development/test_predict_top_ms2deepscores.py +++ b/tests/test_ms2query_development/test_predict_top_ms2deepscores.py @@ -1,7 +1,7 @@ import numpy as np import pytest from ms2query.ms2query_development.AnnotatedSpectrumSet import AnnotatedSpectrumSet -from ms2query.ms2query_development.predict_top_k_ms2deepscore import ( +from ms2query.ms2query_development.reference_methods.predict_top_k_ms2deepscore import ( predict_top_k_ms2deepscores, select_inchikeys_with_highest_ms2deepscore, ) From 21f0a60c1e0ffbb499aaffb153e1bcc9f88f6fd5 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Tue, 24 Mar 2026 09:46:30 +0100 Subject: [PATCH 22/25] Rename test_ReferenceLibrary --- .../{test_run_ms2query.py => test_ReferenceLibrary.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/test_ms2query_development/{test_run_ms2query.py => test_ReferenceLibrary.py} (100%) diff --git a/tests/test_ms2query_development/test_run_ms2query.py b/tests/test_ms2query_development/test_ReferenceLibrary.py similarity index 100% rename from tests/test_ms2query_development/test_run_ms2query.py rename to tests/test_ms2query_development/test_ReferenceLibrary.py From 3d9a1d8dfe82d0b94e9f391c8c34c8951a28039d Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Tue, 24 Mar 2026 19:02:28 +0100 Subject: [PATCH 23/25] Fix bug --- ms2query/ms2query_development/ReferenceLibrary.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ms2query/ms2query_development/ReferenceLibrary.py b/ms2query/ms2query_development/ReferenceLibrary.py index 4ba5e4a..d9fa9b4 100644 --- a/ms2query/ms2query_development/ReferenceLibrary.py +++ b/ms2query/ms2query_development/ReferenceLibrary.py @@ -68,7 +68,7 @@ def load_from_directory(cls, library_file_directory) -> "ReferenceLibrary": reference_embeddings_file = library_file_directory / cls.embedding_file_name top_k_tanimoto_scores_file = library_file_directory / cls.top_k_tanimoto_scores_file_name reference_metadata_file = library_file_directory / cls.reference_metadata_file_name - ms2deepscore_model_file_name = library_file_directory = cls.ms2deepscore_model_file_name + ms2deepscore_model_file_name = library_file_directory / cls.ms2deepscore_model_file_name return cls.load_from_files( ms2deepscore_model_file_name, reference_embeddings_file, top_k_tanimoto_scores_file, reference_metadata_file ) @@ -99,6 +99,8 @@ def create_from_spectra( """Creates all the files needed for MS2Query and stores them""" if store_file_directory is None: store_file_directory = Path(ms2deepscore_model_file_name).parent + else: + store_file_directory = Path(store_file_directory) if store_files: # Check the files don't exist yet for file in ( From cf4f6976a9737673eeac1bbf954cad923c64da04 Mon Sep 17 00:00:00 2001 From: niekdejonge Date: Tue, 24 Mar 2026 19:16:24 +0100 Subject: [PATCH 24/25] Add tutorial that did actually run --- ms2query/notebooks/tutorial.ipynb | 1021 +++++++++++++++++++++++++++-- 1 file changed, 967 insertions(+), 54 deletions(-) diff --git a/ms2query/notebooks/tutorial.ipynb b/ms2query/notebooks/tutorial.ipynb index 411561f..7afc892 100644 --- a/ms2query/notebooks/tutorial.ipynb +++ b/ms2query/notebooks/tutorial.ipynb @@ -10,29 +10,18 @@ }, { "cell_type": "markdown", - "id": "4d564033", + "id": "3fdc846e", "metadata": {}, "source": [ - "# optional: download a matchms cleaned library\n", - "The code below downloads an already matchms cleaned library and an MS2DeepScore model. You can also use your own library, but make sure you know what you are doing and clean the library first. If you just have a few reference spectra, it is probably best to combine your spectra with the reference spectra below to make sure the MS2Query search works properly. " + "### Download already created library" ] }, { "cell_type": "code", - "execution_count": null, - "id": "b7df418e", + "execution_count": 57, + "id": "5ce3469d", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The file ./zenodo_files\\data_split_inchikeys.json already exists, the file won't be downloaded\n", - "The file ./zenodo_files\\merged_and_cleaned_libraries_1.mgf already exists, the file won't be downloaded\n", - "The file ./zenodo_files\\ms2deepscore_model.pt already exists, the file won't be downloaded\n" - ] - } - ], + "outputs": [], "source": [ "import requests\n", "import os\n", @@ -50,34 +39,451 @@ " if chunk:\n", " f.write(chunk)\n", " bar.update(len(chunk)) # Update progress bar by the chunk size\n", - "folder_to_store_zenodo_files = \"./zenodo_files\"\n", - "os.makedirs(folder_to_store_zenodo_files, exist_ok=True)\n", + "folder_to_store_zenodo_files = \"./zenodo_files/full_library\"\n", + "os.makedirs(folder_to_store_zenodo_files, exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "125332e2-fb26-45a4-b649-b84a7ed88181", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The file ./zenodo_files/full_library/embeddings.npz already exists, the file won't be downloaded\n", + "The file ./zenodo_files/full_library/library_metadata.parquet already exists, the file won't be downloaded\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Downloading\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The file ./zenodo_files/full_library/top_k_tanimoto_scores.parquet already exists, the file won't be downloaded\n" + ] + } + ], + "source": [ + "download_file(\"https://zenodo.org/records/19206616/files/embeddings.npz?download=1\", \n", + " os.path.join(folder_to_store_zenodo_files, \"embeddings.npz\"))\n", + "download_file(\"https://zenodo.org/records/19206616/files/library_metadata.parquet?download=1\", \n", + " os.path.join(folder_to_store_zenodo_files, \"library_metadata.parquet\"))\n", + "download_file(\"https://zenodo.org/records/19206616/files/ms2deepscore_model?download=1\", \n", + " os.path.join(folder_to_store_zenodo_files, \"ms2deepscore_model.pt\"))\n", + "download_file(\"https://zenodo.org/records/19206616/files/top_k_tanimoto_scores.parquet?download=1\", \n", + " os.path.join(folder_to_store_zenodo_files, \"top_k_tanimoto_scores.parquet\"))" + ] + }, + { + "cell_type": "markdown", + "id": "aa436a42-d850-4ce2-86c5-741646c8b168", + "metadata": {}, + "source": [ + "# initialize reference library" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "8b477da9", + "metadata": {}, + "outputs": [], + "source": [ + "from ms2query.ms2query_development.ReferenceLibrary import ReferenceLibrary\n", + "from pathlib import Path\n", + "reference_library = ReferenceLibrary.load_from_files(os.path.join(folder_to_store_zenodo_files, \"ms2deepscore_model.pt\"),\n", + " os.path.join(folder_to_store_zenodo_files, \"embeddings.npz\"),\n", + " os.path.join(folder_to_store_zenodo_files, \"top_k_tanimoto_scores.parquet\"),\n", + " os.path.join(folder_to_store_zenodo_files, \"library_metadata.parquet\"))\n", "\n", + "# Or directly find the files from a directory\n", + "# reference_library = ReferenceLibrary.load_from_directory(Path(folder_to_store_zenodo_files))" + ] + }, + { + "cell_type": "markdown", + "id": "b637cbac-c79b-4df3-8ad9-0db400a32fb4", + "metadata": {}, + "source": [ + "# Run MS2Query" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "ae96dd88-c8c0-4be0-9355-b29a8f5ce37b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "10it [00:00, 83.01it/s]\n" + ] + } + ], + "source": [ + "test_spectra = list(tqdm(load_from_mgf(\"../../tests/test_data/10_spectra.mgf\")))" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "ec952eaa-102f-4d86-8fbb-b91973f395a2", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Hashing spe\n", + "Computing s\n", + "Predicting \n", + "/lustre/BIF/nobackup/jonge094/ms2query_2/ms2query/ms2query_development/ReferenceLibrary.py:168: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " # construct results df\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
precursor_mzretention_timecollision_energycompound_namesmilesinchikeyspectrum_hashesms2query_reliability_prediction
29144181.06100NaNNoneNicotinuric acidO=C(O)CNC(=O)c1cccnc1ZBSGKPYXQINNGF-UHFFFAOYSA-N4538586676799711060475687385406365205312522933...0.524193
29138181.06100NaNNoneNicotinuric acidO=C(O)CNC(=O)c1cccnc1ZBSGKPYXQINNGF-UHFFFAOYSA-N4691175720354550216218758199469384349519880571...0.477547
29139181.06100NaNNoneNicotinuric acidO=C(O)CNC(=O)c1cccnc1ZBSGKPYXQINNGF-UHFFFAOYSA-N8126043578764904179452276022324867195623210187...0.525872
23334181.00700NaNNonethieno[2,3-d]pyrimidine-4-carboxylic acidO=C(O)c1ncnc2sccc12VOXKHPSKDOLCAC-UHFFFAOYSA-N4621371819674604814601843392349447447010638904...0.392072
29146181.06100NaNNoneNicotinuric acidO=C(O)CNC(=O)c1cccnc1ZBSGKPYXQINNGF-UHFFFAOYSA-N4605782118791699534560809552554836164365415626...0.511462
478528202.04300NaNNonethiabendazolec1ccc2[nH]c(-c3cscn3)nc2c1WJCNZQLZVWNLKY-UHFFFAOYSA-N4037303697924680981028453225819741544585843750...0.487617
209484323.13902105.6420.05-[(2,5-dimethylphenoxy)methyl]-N-(4-pyridinyl...Cc1ccc(C)c(OCc2ccc(C(=O)Nc3ccncc3)o2)c1VYTRUHMGFSNSKK-UHFFFAOYSA-N4704480111668309422204405940015522296005033315...0.396629
478527202.04300NaNNonethiabendazolec1ccc2[nH]c(-c3cscn3)nc2c1WJCNZQLZVWNLKY-UHFFFAOYSA-N4354824398973859393735759007513142956866549634...0.472113
912123330.18200NaNNoneN-[(1R,2S)-2-(4-ethoxyphenyl)cyclopropyl]-3,4-...CCOc1ccc(C2CC2NC(=O)N2CC(=O)N(C)C(C)C2)cc1FUDGTDMCXMAZHS-UHFFFAOYSA-N4689876070452517240544538176971004829216063873...0.487231
478527202.04300NaNNonethiabendazolec1ccc2[nH]c(-c3cscn3)nc2c1WJCNZQLZVWNLKY-UHFFFAOYSA-N4354824398973859393735759007513142956866549634...0.490491
\n", + "
" + ], + "text/plain": [ + " precursor_mz retention_time collision_energy \\\n", + "29144 181.06100 NaN None \n", + "29138 181.06100 NaN None \n", + "29139 181.06100 NaN None \n", + "23334 181.00700 NaN None \n", + "29146 181.06100 NaN None \n", + "478528 202.04300 NaN None \n", + "209484 323.13902 105.64 20.0 \n", + "478527 202.04300 NaN None \n", + "912123 330.18200 NaN None \n", + "478527 202.04300 NaN None \n", + "\n", + " compound_name \\\n", + "29144 Nicotinuric acid \n", + "29138 Nicotinuric acid \n", + "29139 Nicotinuric acid \n", + "23334 thieno[2,3-d]pyrimidine-4-carboxylic acid \n", + "29146 Nicotinuric acid \n", + "478528 thiabendazole \n", + "209484 5-[(2,5-dimethylphenoxy)methyl]-N-(4-pyridinyl... \n", + "478527 thiabendazole \n", + "912123 N-[(1R,2S)-2-(4-ethoxyphenyl)cyclopropyl]-3,4-... \n", + "478527 thiabendazole \n", + "\n", + " smiles \\\n", + "29144 O=C(O)CNC(=O)c1cccnc1 \n", + "29138 O=C(O)CNC(=O)c1cccnc1 \n", + "29139 O=C(O)CNC(=O)c1cccnc1 \n", + "23334 O=C(O)c1ncnc2sccc12 \n", + "29146 O=C(O)CNC(=O)c1cccnc1 \n", + "478528 c1ccc2[nH]c(-c3cscn3)nc2c1 \n", + "209484 Cc1ccc(C)c(OCc2ccc(C(=O)Nc3ccncc3)o2)c1 \n", + "478527 c1ccc2[nH]c(-c3cscn3)nc2c1 \n", + "912123 CCOc1ccc(C2CC2NC(=O)N2CC(=O)N(C)C(C)C2)cc1 \n", + "478527 c1ccc2[nH]c(-c3cscn3)nc2c1 \n", + "\n", + " inchikey \\\n", + "29144 ZBSGKPYXQINNGF-UHFFFAOYSA-N \n", + "29138 ZBSGKPYXQINNGF-UHFFFAOYSA-N \n", + "29139 ZBSGKPYXQINNGF-UHFFFAOYSA-N \n", + "23334 VOXKHPSKDOLCAC-UHFFFAOYSA-N \n", + "29146 ZBSGKPYXQINNGF-UHFFFAOYSA-N \n", + "478528 WJCNZQLZVWNLKY-UHFFFAOYSA-N \n", + "209484 VYTRUHMGFSNSKK-UHFFFAOYSA-N \n", + "478527 WJCNZQLZVWNLKY-UHFFFAOYSA-N \n", + "912123 FUDGTDMCXMAZHS-UHFFFAOYSA-N \n", + "478527 WJCNZQLZVWNLKY-UHFFFAOYSA-N \n", + "\n", + " spectrum_hashes \\\n", + "29144 4538586676799711060475687385406365205312522933... \n", + "29138 4691175720354550216218758199469384349519880571... \n", + "29139 8126043578764904179452276022324867195623210187... \n", + "23334 4621371819674604814601843392349447447010638904... \n", + "29146 4605782118791699534560809552554836164365415626... \n", + "478528 4037303697924680981028453225819741544585843750... \n", + "209484 4704480111668309422204405940015522296005033315... \n", + "478527 4354824398973859393735759007513142956866549634... \n", + "912123 4689876070452517240544538176971004829216063873... \n", + "478527 4354824398973859393735759007513142956866549634... \n", + "\n", + " ms2query_reliability_prediction \n", + "29144 0.524193 \n", + "29138 0.477547 \n", + "29139 0.525872 \n", + "23334 0.392072 \n", + "29146 0.511462 \n", + "478528 0.487617 \n", + "209484 0.396629 \n", + "478527 0.472113 \n", + "912123 0.487231 \n", + "478527 0.490491 " + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "reference_library.run_ms2query(test_spectra)" + ] + }, + { + "cell_type": "markdown", + "id": "cc3f2477", + "metadata": {}, + "source": [ + "# Create a new MS2Query reference library\n", + "Here we show how to create the reference library that can be downloaded from Zenodo. But if you want to make your own of course replace with your own files (no need to download the library)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2501dc1e-9ab0-408d-b869-8cd9874a46d4", + "metadata": {}, + "outputs": [], + "source": [ "download_file(\"https://zenodo.org/records/16882111/files/merged_and_cleaned_libraries_1.mgf?download=1\", \n", " os.path.join(folder_to_store_zenodo_files, \"merged_and_cleaned_libraries_1.mgf\"))\n", "download_file(\"https://zenodo.org/records/17826815/files/ms2deepscore_model.pt?download=1\", \n", " os.path.join(folder_to_store_zenodo_files, \"ms2deepscore_model.pt\"))" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "cae72056-c3d3-4f4f-b976-c47c86002007", + "metadata": {}, + "outputs": [], + "source": [ + "library_spectra = list(tqdm(load_from_mgf(os.path.join(folder_to_store_zenodo_files, \"merged_and_cleaned_libraries_1.mgf\"))))" + ] + }, { "cell_type": "markdown", - "id": "fa99d2dd", + "id": "09c62f09-ac8e-4b06-a286-28b18bf61af8", "metadata": {}, "source": [ - "# Specify file location \n", - "Replace with your file names" + "### Creates the library and saves all the files for future usage. (the files you can also just download above)" ] }, { "cell_type": "code", "execution_count": null, - "id": "6db579a9", + "id": "8067f86d-9120-4611-bae0-5a70159c4328", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Create mapping from inchikey to spectrum: 100%|█████████████████████████████████████████████████████████████████████████████████| 1017531/1017531 [00:05<00:00, 183656.80it/s]\n", + "Hashing spectra: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 1017531/1017531 [01:27<00:00, 11680.81it/s]\n", + "Computing spectral embeddings ...: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 1017531/1017531 [40:32<00:00, 418.26it/s]\n", + "Get most common inchi per inchikey: 100%|████████████████████████████████████████████████████████████████████████████████████████████| 67648/67648 [00:03<00:00, 17833.31it/s]\n", + "Adding fingerprints to Inchikeys: 67648it [10:40, 105.60it/s]\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "\n", + "reference_library = ReferenceLibrary.create_from_spectra(library_spectra, os.path.join(folder_to_store_zenodo_files, \"ms2deepscore_model.pt\"), store_file_directory = Path(os.path.join(folder_to_store_zenodo_files, \"full_library\")))" + ] + }, + { + "cell_type": "markdown", + "id": "dd223f01-e70d-4a0c-8e68-2f28d7746a55", + "metadata": {}, + "source": [ + "# Sanity check that benchmarking resuls also work with new library creation\n", + "Not part of the tutorial, just internal documentation. We test the negative in negative library searching. To check that it matches the original benmarking notebook. " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b7df418e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The file ./zenodo_files/data_split_inchikeys.json already exists, the file won't be downloaded\n", + "The file ./zenodo_files/merged_and_cleaned_libraries_1.mgf already exists, the file won't be downloaded\n", + "The file ./zenodo_files/ms2deepscore_model.pt already exists, the file won't be downloaded\n" + ] + } + ], "source": [ - "library_spectra_file = os.path.join(folder_to_store_zenodo_files, \"merged_and_cleaned_libraries_1.mgf\")\n", - "ms2deepscore_model_file_name = os.path.join(folder_to_store_zenodo_files, \"ms2deepscore_model.pt\")\n", - "query_spectrum_file = \"replace_with_your_lib_spectra.mgf\"" + "os.makedirs(folder_to_store_zenodo_files, exist_ok=True)\n", + "# This file contains the inchikeys used for the data split. This ensures reproducing the exact same results, \n", + "# alternatively you can of course do the data split yourself, but results might differ slightly. \n", + "download_file(\"https://zenodo.org/records/18801895/files/data_split_inchikeys.json?download=1\", \n", + " os.path.join(folder_to_store_zenodo_files, \"data_split_inchikeys.json\"))\n", + "download_file(\"https://zenodo.org/records/16882111/files/merged_and_cleaned_libraries_1.mgf?download=1\", \n", + " os.path.join(folder_to_store_zenodo_files, \"merged_and_cleaned_libraries_1.mgf\"))\n", + "download_file(\"https://zenodo.org/records/17826815/files/ms2deepscore_model.pt?download=1\", \n", + " os.path.join(folder_to_store_zenodo_files, \"ms2deepscore_model.pt\"))" ] }, { @@ -90,91 +496,598 @@ "name": "stderr", "output_type": "stream", "text": [ - "1017531it [09:51, 1720.67it/s]\n" + "730577it [04:27, 2631.43it/s]" ] } ], "source": [ "from matchms.importing import load_from_mgf\n", "from tqdm import tqdm\n", + "import os\n", + "library_spectra = list(tqdm(load_from_mgf(os.path.join(folder_to_store_zenodo_files, \"merged_and_cleaned_libraries_1.mgf\"))))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25be7410", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", "\n", - "library_spectra = list(tqdm(load_from_mgf(library_spectra_file)))\n", - "query_spectra = list(tqdm(load_from_mgf(query_spectrum_file)))" + "with open(os.path.join(folder_to_store_zenodo_files, \"data_split_inchikeys.json\"), \"r\") as f:\n", + " data_split = json.load(f)" ] }, { - "cell_type": "markdown", - "id": "64534dd6", + "cell_type": "code", + "execution_count": null, + "id": "27a82f53", "metadata": {}, + "outputs": [], "source": [ - "# Create the reference library files\n", - "The code below will precompute everything needed to run MS2Query. It will save this in the same folder as your ms2deepscore model. \n", - "The files created are \"embeddings.npz\", \"top_k_tanimoto_scores.parquet\", \"library_metadata.parquet\". " + "val_inchikeys = set(data_split[\"val\"])\n", + "train_inchikeys = set(data_split[\"train\"])\n", + "test_inchikeys = set(data_split[\"test\"])" ] }, { "cell_type": "code", "execution_count": null, - "id": "669a489a", + "id": "bf5b0447", "metadata": {}, "outputs": [], "source": [ - "from ms2query.ms2query_development.ReferenceLibrary import ReferenceLibrary\n", - "reference_library = ReferenceLibrary.create_from_spectra(library_spectra, ms2deepscore_model_file_name)" + "val_spectra = []\n", + "test_spectra = []\n", + "train_spectra = []\n", + "for spectrum in tqdm(library_spectra):\n", + " inchikey = spectrum.get(\"inchikey\")[:14]\n", + " if inchikey in train_inchikeys:\n", + " train_spectra.append(spectrum)\n", + " elif inchikey in val_inchikeys:\n", + " val_spectra.append(spectrum)\n", + " elif inchikey in test_inchikeys:\n", + " test_spectra.append(spectrum)\n", + " else:\n", + " raise ValueError(f\"The inchikey {inchikey} does not appear in the val, test or train inchikeys\")" ] }, { - "cell_type": "markdown", - "id": "dfd0a633", + "cell_type": "code", + "execution_count": 19, + "id": "76471c2c-d7ab-4058-a42b-31b975e250ba", "metadata": {}, + "outputs": [], "source": [ - "# Run MS2Query\n", - "The code above only has to be run once after that you can load the library faster from the saved files. " + "def split_on_ionmode(spectra):\n", + " pos_spectra = []\n", + " neg_spectra = []\n", + " for spectrum in spectra:\n", + " ionmode = spectrum.get(\"ionmode\")\n", + " if ionmode == \"positive\":\n", + " pos_spectra.append(spectrum)\n", + " elif ionmode == \"negative\":\n", + " neg_spectra.append(spectrum)\n", + " else:\n", + " raise ValueError(f\"The ionmode {ionmode} is not positive or negative\")\n", + " return pos_spectra, neg_spectra" ] }, { "cell_type": "code", - "execution_count": null, - "id": "e6cf44e2", + "execution_count": 20, + "id": "02be88d5-3962-4c8e-89ab-72fa7163d47c", "metadata": {}, "outputs": [], "source": [ - "# no need to run if you just created the libary above\n", - "reference_library = ReferenceLibrary.load_from_directory(folder_to_store_zenodo_files)" + "pos_val_spectra, neg_val_spectra = split_on_ionmode(val_spectra)\n", + "pos_test_spectra, neg_test_spectra = split_on_ionmode(test_spectra)\n", + "pos_train_spectra, neg_train_spectra = split_on_ionmode(train_spectra)" ] }, { "cell_type": "code", "execution_count": null, - "id": "566b2b0d", + "id": "669a489a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Create mapping from inchikey to spectrum: 100%|███████████████████████████████████████████████████████████████████████████████████| 283790/283790 [00:01<00:00, 282229.91it/s]\n", + "Hashing spectra: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████| 283790/283790 [00:19<00:00, 14677.55it/s]\n", + "Computing spectral embeddings ...: 11%|██████████ | 30305/283790 [00:31<04:24, 958.90it/s]" + ] + } + ], + "source": [ + "from ms2query.ms2query_development.ReferenceLibrary import ReferenceLibrary\n", + "reference_library = ReferenceLibrary.create_from_spectra(neg_train_spectra, os.path.join(folder_to_store_zenodo_files, \"ms2deepscore_model.pt\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "829714f0", "metadata": {}, "outputs": [], "source": [ - "results = reference_library.run_ms2query(query_spectra)" + "results = reference_library.run_ms2query(neg_val_spectra)" ] }, { - "cell_type": "markdown", - "id": "98ce9660", + "cell_type": "code", + "execution_count": 27, + "id": "c9f79c50-0c19-4a9c-afd1-115233581193", "metadata": {}, + "outputs": [], "source": [ - "print(results)" + "results.to_csv(os.path.join(folder_to_store_zenodo_files, \"neg_neg_results.csv\"))" ] }, { "cell_type": "code", - "execution_count": null, - "id": "b7e3b533", + "execution_count": 28, + "id": "49771619-f9ed-46a1-8b9f-5ae70377fd7d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
precursor_mzretention_timecollision_energycompound_namesmilesinchikeyspectrum_hashesms2query_reliability_prediction
4574266.0589469.3530.0379724-54-6O=C(O)C1CCN(C(=O)c2ccc(Cl)cc2)CC1FGLQAKJRFPNIJT-UHFFFAOYSA-N4287947218867478945721777143654811527279348390...0.739373
4576266.0589469.3560.0379724-54-6O=C(O)C1CCN(C(=O)c2ccc(Cl)cc2)CC1FGLQAKJRFPNIJT-UHFFFAOYSA-N8445518326529625949101509334596600018097006285...0.605217
4574266.0589469.3530.0379724-54-6O=C(O)C1CCN(C(=O)c2ccc(Cl)cc2)CC1FGLQAKJRFPNIJT-UHFFFAOYSA-N4287947218867478945721777143654811527279348390...0.739373
4578266.05900NaNNone379724-54-6O=C(O)C1CCN(C(=O)c2ccc(Cl)cc2)CC1FGLQAKJRFPNIJT-UHFFFAOYSA-N4453904535900060540047468210876322356832506165...0.759073
4581266.05900NaNNone379724-54-6O=C(O)C1CCN(C(=O)c2ccc(Cl)cc2)CC1FGLQAKJRFPNIJT-UHFFFAOYSA-N4438257318628859128622642248978796499540902384...0.770879
...........................
283597740.52030NaN45.0PE 36:3CCCCCC=CCC=CCCCCCCCC(=O)OC(COC(=O)CCCCCCCC=CCC...GKAFCSRKMWFPSJ-UHFFFAOYSA-N4605131291822750236269081691830926191534825838...0.943663
283597740.52030NaN45.0PE 36:3CCCCCC=CCC=CCCCCCCCC(=O)OC(COC(=O)CCCCCCCC=CCC...GKAFCSRKMWFPSJ-UHFFFAOYSA-N4605131291822750236269081691830926191534825838...0.943663
201773744.55300NaN20.0PE(18:0_18:1) - (2-aminoethoxy)[2-[octadec-9-e...CCCCCCCCC=CCCCCCCCC(=O)OC(COC(=O)CCCCCCCCCCCCC...JQKOHRZNEOQNJE-UHFFFAOYSA-N8543555785430847640307667357763008254598100629...0.921203
235132766.54100NaN30.01-Octadecanoyl-2-(5Z,8Z,11Z,14Z-eicosatetraeno...CCCCCC=CCC=CCC=CCC=CCCCC(=O)OC(COC(=O)CCCCCCCC...ANRKEHNWXKCXDB-UHFFFAOYSA-N4772335068029728878564587931928277302571040079...0.907530
235132766.54100NaN30.01-Octadecanoyl-2-(5Z,8Z,11Z,14Z-eicosatetraeno...CCCCCC=CCC=CCC=CCC=CCCCC(=O)OC(COC(=O)CCCCCCCC...ANRKEHNWXKCXDB-UHFFFAOYSA-N4772335068029728878564587931928277302571040079...0.907530
\n", + "

15967 rows × 8 columns

\n", + "
" + ], + "text/plain": [ + " precursor_mz retention_time collision_energy \\\n", + "4574 266.05894 69.35 30.0 \n", + "4576 266.05894 69.35 60.0 \n", + "4574 266.05894 69.35 30.0 \n", + "4578 266.05900 NaN None \n", + "4581 266.05900 NaN None \n", + "... ... ... ... \n", + "283597 740.52030 NaN 45.0 \n", + "283597 740.52030 NaN 45.0 \n", + "201773 744.55300 NaN 20.0 \n", + "235132 766.54100 NaN 30.0 \n", + "235132 766.54100 NaN 30.0 \n", + "\n", + " compound_name \\\n", + "4574 379724-54-6 \n", + "4576 379724-54-6 \n", + "4574 379724-54-6 \n", + "4578 379724-54-6 \n", + "4581 379724-54-6 \n", + "... ... \n", + "283597 PE 36:3 \n", + "283597 PE 36:3 \n", + "201773 PE(18:0_18:1) - (2-aminoethoxy)[2-[octadec-9-e... \n", + "235132 1-Octadecanoyl-2-(5Z,8Z,11Z,14Z-eicosatetraeno... \n", + "235132 1-Octadecanoyl-2-(5Z,8Z,11Z,14Z-eicosatetraeno... \n", + "\n", + " smiles \\\n", + "4574 O=C(O)C1CCN(C(=O)c2ccc(Cl)cc2)CC1 \n", + "4576 O=C(O)C1CCN(C(=O)c2ccc(Cl)cc2)CC1 \n", + "4574 O=C(O)C1CCN(C(=O)c2ccc(Cl)cc2)CC1 \n", + "4578 O=C(O)C1CCN(C(=O)c2ccc(Cl)cc2)CC1 \n", + "4581 O=C(O)C1CCN(C(=O)c2ccc(Cl)cc2)CC1 \n", + "... ... \n", + "283597 CCCCCC=CCC=CCCCCCCCC(=O)OC(COC(=O)CCCCCCCC=CCC... \n", + "283597 CCCCCC=CCC=CCCCCCCCC(=O)OC(COC(=O)CCCCCCCC=CCC... \n", + "201773 CCCCCCCCC=CCCCCCCCC(=O)OC(COC(=O)CCCCCCCCCCCCC... \n", + "235132 CCCCCC=CCC=CCC=CCC=CCCCC(=O)OC(COC(=O)CCCCCCCC... \n", + "235132 CCCCCC=CCC=CCC=CCC=CCCCC(=O)OC(COC(=O)CCCCCCCC... \n", + "\n", + " inchikey \\\n", + "4574 FGLQAKJRFPNIJT-UHFFFAOYSA-N \n", + "4576 FGLQAKJRFPNIJT-UHFFFAOYSA-N \n", + "4574 FGLQAKJRFPNIJT-UHFFFAOYSA-N \n", + "4578 FGLQAKJRFPNIJT-UHFFFAOYSA-N \n", + "4581 FGLQAKJRFPNIJT-UHFFFAOYSA-N \n", + "... ... \n", + "283597 GKAFCSRKMWFPSJ-UHFFFAOYSA-N \n", + "283597 GKAFCSRKMWFPSJ-UHFFFAOYSA-N \n", + "201773 JQKOHRZNEOQNJE-UHFFFAOYSA-N \n", + "235132 ANRKEHNWXKCXDB-UHFFFAOYSA-N \n", + "235132 ANRKEHNWXKCXDB-UHFFFAOYSA-N \n", + "\n", + " spectrum_hashes \\\n", + "4574 4287947218867478945721777143654811527279348390... \n", + "4576 8445518326529625949101509334596600018097006285... \n", + "4574 4287947218867478945721777143654811527279348390... \n", + "4578 4453904535900060540047468210876322356832506165... \n", + "4581 4438257318628859128622642248978796499540902384... \n", + "... ... \n", + "283597 4605131291822750236269081691830926191534825838... \n", + "283597 4605131291822750236269081691830926191534825838... \n", + "201773 8543555785430847640307667357763008254598100629... \n", + "235132 4772335068029728878564587931928277302571040079... \n", + "235132 4772335068029728878564587931928277302571040079... \n", + "\n", + " ms2query_reliability_prediction \n", + "4574 0.739373 \n", + "4576 0.605217 \n", + "4574 0.739373 \n", + "4578 0.759073 \n", + "4581 0.770879 \n", + "... ... \n", + "283597 0.943663 \n", + "283597 0.943663 \n", + "201773 0.921203 \n", + "235132 0.907530 \n", + "235132 0.907530 \n", + "\n", + "[15967 rows x 8 columns]" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "6d9d10f8-a042-48c2-a9d1-b925815b9ac0", + "metadata": {}, + "outputs": [], + "source": [ + "predicted_inchikeys = [inchikey[:14] for inchikey in results[\"inchikey\"].tolist()]" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "de56ab2f-1e81-462a-9197-7d6099c22ad3", "metadata": {}, "outputs": [], "source": [ - "results.to_csv(\"ms2query_results.csv\")" + "predicted_scores = results[\"ms2query_reliability_prediction\"].tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "b4bd52eb-742e-4a0b-bdaf-77b7a182a4e3", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "import os\n", + "intermediates_folder = \"./cross_ionmode_intermediate\"\n", + "\n", + "with open(os.path.join(intermediates_folder, \"all_fingerprints.pickle\"), \"rb\") as handle:\n", + " all_fingerprints = pickle.load(handle)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "1905a382-d0d5-4c45-a2c6-11358e509e70", + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import tqdm\n", + "from matchms.similarity.vector_similarity_functions import jaccard_similarity_matrix\n", + "import numpy as np\n", + "def get_prediction_tanimoto(predicted_inchikeys, val_spectra, val_and_train_fingerprints):\n", + " assert len(predicted_inchikeys) == len(val_spectra)\n", + "\n", + " tanimoto_for_prediction = []\n", + " for index, predicted_inchikey in enumerate(tqdm(predicted_inchikeys)):\n", + " if predicted_inchikey is not None:\n", + " correct_inchikey = val_spectra[index].get(\"inchikey\")[:14]\n", + " predicted_fingerprint = val_and_train_fingerprints.get_fingerprints([predicted_inchikey])[0]\n", + " actual_fingerprint = val_and_train_fingerprints.get_fingerprints([correct_inchikey])[0]\n", + " tanimoto_prediction = jaccard_similarity_matrix(np.array([predicted_fingerprint]), np.array([actual_fingerprint]))[0][0]\n", + " tanimoto_for_prediction.append(tanimoto_prediction)\n", + " else:\n", + " tanimoto_for_prediction.append(0.0)\n", + " return tanimoto_for_prediction" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "1ec0c831-ba10-4c0f-907e-6b7420f35ebd", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████| 15967/15967 [00:00<00:00, 18131.11it/s]\n" + ] + } + ], + "source": [ + "neg_neg_tanimoto_for_prediction = get_prediction_tanimoto(predicted_inchikeys, neg_val_spectra, all_fingerprints)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "7eeccfdc-1542-43fc-ba7c-79ece4346a4d", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "def create_values_violin_plot(predicted_scores, val_spectra, tanimoto_for_predictions, n_buckets):\n", + " \"\"\"The predictions are ranked based on the predicted scores, and are split in different groups from high to low, \n", + " for each group the average tanimoto per inchikey is computed.\"\"\"\n", + " tanimoto_per_bucket = []\n", + " ranked_indexes_per_bucket = bucket_by_value(predicted_scores, n_buckets)\n", + " for indexes_in_bucket in ranked_indexes_per_bucket: \n", + " average_tanimoto_per_inchikey = get_average_tanimoto_per_inchikey(val_spectra, tanimoto_for_predictions, indexes_in_bucket)\n", + " tanimoto_per_bucket.append(average_tanimoto_per_inchikey)\n", + " return tanimoto_per_bucket\n", + "\n", + "def bucket_by_value(values, n_buckets=10):\n", + " \"\"\"\n", + " Splits values into n equally sized buckets based on rank. \n", + " So the highest scores are in the first bucket en the lowest in the last. Each bucket has the same number of values. \n", + " \"\"\"\n", + " values = np.array(values)\n", + " # Get indexes that would sort the array descending (top values first)\n", + " sorted_indexes = np.argsort(values)[::-1]\n", + " \n", + " # Split sorted indexes into n equal(ish) buckets\n", + " buckets = np.array_split(sorted_indexes, n_buckets)\n", + " \n", + " return buckets\n", + " \n", + "def get_average_tanimoto_per_inchikey(query_spectrum_set, tanimoto_for_predictions, selected_indexes):\n", + " \"\"\"Calculates the average tanimoto per inchikey for the selected indexes\"\"\"\n", + " assert len(tanimoto_for_predictions) == len(query_spectrum_set)\n", + " average_scores_per_inchikey = []\n", + " # Calculate score per unique inchikey\n", + " for inchikey in query_spectrum_set.spectrum_indices_per_inchikey.keys():\n", + " matching_spectrum_indexes = query_spectrum_set.spectrum_indices_per_inchikey[inchikey]\n", + " prediction_scores = []\n", + " for index in matching_spectrum_indexes:\n", + " if index in selected_indexes:\n", + " prediction_scores.append(tanimoto_for_predictions[index])\n", + " if len(prediction_scores) > 0:\n", + " average_prediction = sum(prediction_scores) / len(prediction_scores)\n", + " average_scores_per_inchikey.append(average_prediction)\n", + " return average_scores_per_inchikey\n", + "\n", + "def plot_average_per_threshold(predicted_scores, tanimoto_for_predictions, val_spectra, n_of_buckets):\n", + " buckets = bucket_by_value(predicted_scores,n_of_buckets)\n", + " mean_per_fraction = []\n", + " selected_indexes = set()\n", + " for indexes_in_bucket in tqdm(buckets):\n", + " selected_indexes.update(set(indexes_in_bucket.tolist()))\n", + " average_scores_per_inchikey = get_average_tanimoto_per_inchikey(val_spectra, tanimoto_for_predictions, selected_indexes)\n", + " mean = sum(average_scores_per_inchikey)/len(average_scores_per_inchikey)\n", + " mean_per_fraction.append(mean)\n", + " return mean_per_fraction\n", + " \n", + "from matplotlib import pyplot as plt\n", + "\n", + "def plot_comparison_plot(results):\n", + " n = len(list(results.values())[0])\n", + " labels = [f\"0 - {(100/n)*(i+1)} %\" for i in range(n)]\n", + " for label, mean_per_fraction in results.items():\n", + " plt.plot(labels, mean_per_fraction, label = label)\n", + " \n", + " plt.xlabel(\"Fraction predicted\")\n", + " plt.ylabel(\"Average Tanimoto\")\n", + " plt.legend()\n", + " plt.ylim((0, 1))\n", + " \n", + " step = n // 10\n", + " positions = range(len(labels))\n", + " \n", + " plt.xticks(\n", + " ticks=positions[step-1::step], # start at 9, then 19, 29, ...\n", + " labels=labels[step-1::step],\n", + " rotation=45,\n", + " ha='right'\n", + " )\n", + " plt.grid(axis='both', linestyle='--', alpha=0.5)\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "3bf6e470-aa36-4c07-85d0-474d0e52002b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Create mapping from inchikey to spectrum: 100%|████████████████████████████████████| 15967/15967 [00:00<00:00, 367569.63it/s]\n", + "100%|█████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 603.54it/s]\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArMAAALgCAYAAACH/f9CAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjcsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvTLEjVAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAqoVJREFUeJzs3Xd8VFX+//H3vTOZCZk0QkiBBAgQQLpUQQVdUVRW5WvXVRTr2oW1r8riruK6ytr46a6KDXsDFRQRBUUQpAkokJACSUgvk2RSJjP3/v4IM0saZJI7ufeevJ+PBw/1MknOyzvGT4Z7z0iqqqogIiIiIjIhWe8FEBERERF1FIdZIiIiIjItDrNEREREZFocZomIiIjItDjMEhEREZFpcZglIiIiItPiMEtEREREpsVhloiIiIhMi8MsEREREZkWh1kiIiIiMi1dh9kffvgB5513Hvr06QNJkrB8+fLjfsy6deswbtw42O12DB48GG+88UbQ10lERERExqTrMOtyuTBmzBgsWbKkXY/PysrCrFmzcPrpp2Pnzp24++67ccMNN2D16tVBXikRERERGZGkqqqq9yIAQJIkfPbZZ5g9e3abj7n//vuxcuVK7Nmzx3/s8ssvR0VFBb7++usuWCURERERGYlV7wUEYtOmTZgxY0aTYzNnzsTdd9/d5sfU19ejvr7e/8+KoqCsrAy9evWCJEnBWioRERERdZCqqqiqqkKfPn0gy8e+kMBUw2xBQQHi4+ObHIuPj0dlZSVqa2vRo0ePFh+zaNEiLFy4sKuWSEREREQaycnJQVJS0jEfY6phtiMefPBBzJ8/3//PTqcT/fr1Q3Z2NiIjIwE0XuIgyzIURcHRV120dVyWZUiS1OZxr9fbZA2+nygURWnXcYvFAlVVmxz3raWt4+1de2ebvF4vMjMzMXjwYFgsFiGamq+9oaEBmZmZGDhwICwWixBNzdeuqioyMjKQkpICi8UiRNPRx33P04EDB8JmswnRdDSLxQKPx4MDBw74n6ciNDVfu6IoyMrKwsCBA5v8SZqZm5qfJ99zddCgQQgJCRGiqfnxhoYGZGRktHiumrmp+Vp8z9WUlJQmryKauan58eM9V7VuqqiowIABAxAREYHjMdUwm5CQgMLCwibHCgsLERkZ2eqrsgBgt9tht9tbHO/Zs6d/mKX283q9iIyMRFRUVJMhSCS+xp49ewrdGBERIWxjdzmH3aExPDy8W3y/iY6OFr6xOzxXu8N57KpG3w+w7bkk1FT7zE6ZMgVr165tcmzNmjWYMmWKTisiIiIiIj3puptBdXU1Dhw4AAA48cQTsXjxYpx++umIiYlBv3798OCDDyIvLw9vvfUWgMatuUaOHInbbrsN1113Hb777jvceeedWLlyJWbOnNmur1lZWYmoqCg4nU6+MtsBvj+e8P2xgYjYaH6i9wFsFAUbxcBG7QUyr+n6yuzWrVtx4okn4sQTTwQAzJ8/HyeeeCIeffRRAEB+fj4OHTrkf3xKSgpWrlyJNWvWYMyYMXjmmWfw6quvtnuQJW14PB69lxB0bDQ/0fsANoqCjWJgo34Ms89sV+Ers53j9XqRnp6O1NRUoa8LYqO5id4HsFEUZm1UVRUej6fFTT2t8Xq9OHjwIPr372+qxkCwsWNCQkLa/FyBzGumugGMiIiI9OV2u5Gfn4+ampp2Pd43+B48eFDoP4JnY+AkSUJSUhLCw8M79Xk4zBIREVG7+Lagslgs6NOnD2w223EHG1VVUV9fD7vdLvSgx8bAP19xcTFyc3M7/ScTHGYpYMd7Jw4RsNH8RO8D2CgKMzW63W4oioLk5GSEhYW162NUVYUkScIPemwMXO/evZGdnY2GhoZODbO8ZpaIiIjapa6uzv/mAKGhoXovh0zuWM8n0+xmQOajqiqqq6sh8s9AbDQ/0fsANoqiuzR6vV42mpyRGznMUkAURUFubm6Lt58TCRvNT/Q+gI2i6A6NQOPlCaJjo344zBIRERGRaXGYJSIiIuFde+21kCQJf/7zn1v83m233QZJknDttdf6jxUXF+OWW25Bv379YLfbkZCQgJkzZ+Knn34CAJSVleGOO+7A0KFDERYWhiFDhuDOO++E0+ls8fnffPNNTJw4EWFhYYiIiMD06dPx5ZdfBq1VC4sWLcLEiRMRERGBuLg4/N///R/S0tKO+3EfffQRhg0bhtDQUIwaNQqrVq0K+lo5zFJAJElq11YsZsZG8xO9D2CjKLpDI2CcHRuSk5Px/vvvo7a21n+srq4O7777Lvr169fksRdddBF27NiBN998E2lpafj8889x2mmnobS0FABw+PBhHD58GE8//TR2796NV155BatXr8b111/f5PPcc889uPnmm3HZZZdh165d2LJlC0455RRccMEFePHFF4Pe3NDQ0KGPW79+PW677Tb8/PPPWLNmDRoaGnDeeefB5XK1+TEbN27EFVdcgeuvvx47duzA7NmzMXv2bOzZs6ejy28ftZtxOp0qANXpdOq9FCIiIlOpra1Vf//9d7W2ttZ/TFEU1VXfoMsvRVHavfZrrrlGveCCC9SRI0eqy5Yt8x9/55131NGjR6sXXHCBes0116iqqqrl5eUqAHXdunUB/fv58MMPVZvNpjY0NKiqqqqbNm1SAajPP/98i8fOnz9fDQkJUQ8dOqSqqqouWLBAHTNmTJPH/Pvf/1b79+/f5Ngrr7yiDhs2TLXb7erQoUPVJUuW+H8vKytLBaC+//776rRp01S73a6++OKLakREhPrRRx81+TyfffaZGhYWplZWVrarraioSAWgrl+/vs3HXHrppeqsWbOaHJs8ebJ68803t/r41p5PPoHMa9xnlgKiqiqcTieioqKEfSWBjeYneh/ARlGI0Fjb4MXwR1fr8rV/f2wmwmyBjTLXXXcdXn/9dfzpT38CACxduhRz587FunXr/I8JDw9HeHg4li9fjpNOOgl2u/2Yn1M9cqd/RUUFIiMjYbU2rum9995DeHg4br755hYf85e//AWLFy/GJ598grvvvrtda3/nnXfw6KOP4sUXX8SJJ56IHTt24MYbb4TD4cA111zjf9wDDzyAZ555BieeeCJCQ0Px66+/4vXXX8fFF1/sf4zvnyMiItr1tSsqKgAAPXv2bPMxmzZtwvz585scmzlzJpYvX96ur9FRxnjdn0xDURQUFBQIfectG81P9D6AjaLoDo1Gc9VVV2HDhg04ePAgDh48iJ9++glXXXVVk8dYrVa88cYbePPNNxEdHY2TTz4ZDz30EHbt2tXm583Pz8c//vEP3HTTTf5jaWlpGDRoEGw2W4vH9+nTB5GRke26DtVnwYIFeOaZZ3DhhRciJSUFF154IebNm4f//Oc/TR539913+x+TmJiIG264AatXr0Z+fj4AoKioCKtWrcJ1113Xrq+rKArmzZuHKVOmYOTIkW0+rqCgAPHx8U2OxcfHo6CgoN2NHcFXZomIiKjDeoRY8PtjM9v8fVVVUVdXj9BQ7d8dq0dI4O8a1bt3b8yaNQtvvPEGVFXFrFmzEBsb2+JxF110EWbNmoUff/wRP//8M7766is89dRTePXVV5vcKAY0bvB/4YUXYvjw4fjb3/7W5PfU4+zL2tqg2xqXy4WMjAxcf/31uPHGG/3HPR4PoqKimjx2woQJTf550qRJGDFiBN5880088MADWLZsGfr3749p06a162vfdttt2LNnD9asWdOux3c1DrNERETUYZIkHfOP+lVVhax4EGqzGuZSiuuuuw633347AGDJkiVtPi40NBRnnnkmzjzzTDzyyCO44YYbsGDBgibDbFVVFc455xxERETg008/RUhIiP/3UlNTsWHDBrjd7hZD6+HDh1FZWYkhQ4YAaLxJrvnge/TNW9XV1QCAV155BZMnT27yuOZvBetwOFq03HDDDViyZAkeeOABvP7665g7d267zsftt9+OL7/8EuvXr0diYuIxH5uQkIDCwsImxwoLC5GQkHDcr9MZvMyAAiJJEhwOh2G+IQUDG81P9D6AjaLoDo1Ay2FLb2effTbcbjcaGhowc2bbryo3N3z48CZ381dWVuKss86CzWbDp59+2uItWa+44gpUV1e3uAwAAJ5++mmEhobisssuA9D4inFBQUGTgXbnzp3+v4+Pj0efPn2QmZmJwYMHN/mVkpJy3LVfddVVOHjwIJ5//nn8/vvvTa6xbY2qqrj99tvx2Wef4bvvvkNKSspxz+OUKVOwdu3aJsfWrFmDKVOmHHd9ncFXZikgsiwjOTlZ72UEFRvNT/Q+gI2i6A6Nvu3HjMRisWDv3r3+v2+utLQUl1xyCa677jqMHj0aERER2Lp1K5566ilccMEFAP43yNbU1GDZsmWoq6tDXV0dgMbB1GKxYMqUKbjrrrtw7733wu12Y/bs2WhoaMCyZcvw/PPP44033kCvXr0AAKeddhqKi4vx1FNP4eKLL8bXX3+Nr776CpGRkf51LVy4EHfeeSeioqJw9tlno76+Hlu3bkV5eXmLG6+a69mzJy688ELce++9OOuss5CUlHTMx99222149913sWLFCkRERPhfcY2KikKPHj0AAHPmzEHfvn2xaNEiAMBdd92F6dOn45lnnsGsWbPw/vvvY+vWrfjvf/973HPSKcfd70Aw3Jqrc7xer1pcXKx6vV69lxI0bDQ/0ftUlY2iMFvjsbZSaouiKKrb7Q5oG61g8G3N1Zajt+aqq6tTH3jgAXXcuHFqVFSUGhYWpg4dOlR9+OGH1ZqaGlVVVfX7779XAbT6Kysrq8nnfu2119Tx48eroaGhKgDVZrO1usXVSy+9pCYnJ6sOh0OdM2eO+vjjj7fYmuudd95Rx44dq9psNrVnz57qtGnT1E8//VRV1f9tzbVjx45WG9euXasCUD/88MPj/vtqq23p0qX+x0yfPt3/78znww8/VIcMGaLabDZ1xIgR6sqVK9v8GlptzSUdWXC3UVlZiaioKDidziY/7VD7eL1epKenIzU11XB/bKQVNpqf6H0AG0Vhtsa6ujpkZWUhJSWlxR+pt0VVVdTV1SE0NFTYyykCaczOzsb06dMxZcoUvPPOO1163t9++23MmzcPhw8fDvjV8mCcx2M9nwKZ13jNLBEREVEXGTBgANatW4dhw4Y1uSY2mGpqapCRkYEnn3wSN998s+Eu++gsDrNEREREXSglJQV/+9vfMH78+C75ek899RSGDRuGhIQEPPjgg13yNbsSh1kKiCRJpn6nmvZgo/mJ3gewURTdoREw3m4GwWDkxr/97W9oaGjA2rVrER4e3uHPY9RG7mZAAZFl+bj7zJkdG81P9D6AjaLoDo1G3M1Aa2zUF1+ZpYAoioL8/Hyh33qRjeYneh/ARlGYtTGQe8dVVYXb7Q7oY8yGjR3/nFrgMEsBUVUVTqdT+P9g2WhuovcBbBSF2Rp9725VU1MT0Md5vd5gLMdQ2Bg4t9sNoPOXL/AyAyIiImoXi8WC6OhoFBUVAQDCwsKOe72vqqqor68HAGGvDWZj4BRFQXFxMcLCwmC1dm4c5TBLRERE7ZaQkAAA/oH2eFRVhcfjgdVqFXrQY2PgZFlGv379Ov35OMxSQCRJQmxsrLD/sQJsFIHofQAbRWHGRkmSkJiYiLi4ODQ0NBz38YqiwOl0IioqCrIs5tWNbOwYm82myefiO4ARERERkaHwHcAoaBRFQU5OjunuvA0EG81P9D6AjaJgoxjYqC8OsxQQVVXhcrlMc+dtR7DR/ETvA9goCjaKgY364jBLRERERKbFYZaIiIiITIvDLAVElmUkJCQIe7cmwEYRiN4HsFEUbBQDG/XF3QyIiIiIyFC4mwEFjaIoyMzMNOTdjFpho/mJ3gewURRsFAMb9cVhlgKiqircbrch72bUChvNT/Q+gI2iYKMY2KgvDrNEREREZFocZomIiIjItDjMUkBkWUZSUpIh72bUChvNT/Q+gI2iYKMY2Kgv7mZARERERIbC3QwoaLxeL9LS0uD1evVeStCw0fxE7wPYKAo2ioGN+uIwSwEz4rYcWmOj+YneB7BRFGwUAxv1w2GWiIiIiEyLwywRERERmRZvAKOA+DZNttlskCRJ7+UEBRvNT/Q+gI2iYKMY2Kg93gBGQWW1WvVeQtCx0fxE7wPYKAo2ioGN+uEwSwFRFAXp6emGvQhcC2w0P9H7ADaKgo1iYKO+OMwSERERkWlxmCUiIiIi0+IwS0RERESmxd0MKCCqqkJRFMiyLPQdm2w0N9H7ADaKgo1iYKP2uJsBBZXH49F7CUHHRvMTvQ9goyjYKAY26ofDLAVEURRkZWUZ8m5GrbDR/ETvA9goCjaKgY364jBLRERERKbFYZaIiIiITIvDLAVMlsV/2rDR/ETvA9goCjaKgY364W4GRERERGQo3M2AgkZVVVRXV0Pkn4HYaH6i9wFsFAUbxcBGfXGYDbJatxerducj31mr91I0oSgKcnNzDXk3o1bYaH6i9wFsFAUbxcBGfXGYDbJb3tmGW9/ZjhU7D+u9FCIiIiLhcJgNsjOHxwMAvtzFYZaIiIhIaxxmg+zsEQmwyBL25FUiu8Sl93I6TZIk2Gw2Yd+uD2CjCETvA9goCjaKgY364m4GXeDq1zbjx/QS3HPWENz+h9Qu+ZpEREREZsXdDAzmj6MTAQBf7srXeSWdp6oqKioqDHk3o1bYaH6i9wFsFAUbxcBGfXGY7QIzRyTAKkvYV1CFA0VVei+nUxRFQUFBgSHvZtQKG81P9D6AjaJgoxjYqC8Os10gOsyGU1NjAQBf/Gr+V2eJiIiIjILDbBf54+g+AICVu/MN+RI9ERERkRlxmO0iZ46Ih80i40BRNfYXmvdSA0mS4HA4DHk3o1bYaH6i9wFsFAUbxcBGfXE3gy5041tbseb3Qtx++mDcM3Nol35tIiIiIrPgbgYG9b9dDQ6b9lIDRVFQUlJiyAvAtcJG8xO9D2CjKNgoBjbqi8NsF5pxQjxCQ2Rkl9bgt8OVei+nQ1RVRUlJiWmH8fZgo/mJ3gewURRsFAMb9cVhtgs57Fb8YVgcAOALvr0tERERUadxmO1i/l0NdnFXAyIiIqLO4jDbxU4fGocwmwW55bX4Ndep93ICJkkSoqKiDHk3o1bYaH6i9wFsFAUbxcBGfXGY7WI9bBaccUI8AODLX813qYEsy0hMTIQsi/vUYaP5id4HsFEUbBQDG/VlvBV1A75dDVbuzoeimOtSA0VRkJ+fb8i7GbXCRvMTvQ9goyjYKAY26ovDrA6mD+mNcLsV+c46/J5vrl0NVFWF0+kU+npfNpqf6H0AG0XBRjGwUV8cZnUQGmLB2ORoAMBvh8133SwRERGRUXCY1cnwPo3vZvG7SfebJSIiIjICDrM6GZ7YOMya7c0TJElCbGysIe9m1AobzU/0PoCNomCjGNioL6veC+iuRhx5ZXZvfiUURYUsG+/J0RpZlhEbG6v3MoKKjeYneh/ARlGwUQxs1BdfmdVJSqwDdqsMl9uLQ2U1ei+n3RRFQU5OjiHvZtQKG81P9D6AjaJgoxjYqC8OszqxWmQMS4gAYK5LDVRVhcvlMuTdjFpho/mJ3gewURRsFAMb9cVhVkfD+0QBAH7P544GRERERB3BYVZH3NGAiIiIqHM4zOrIjDsayLKMhIQEQ76dnVbYaH6i9wFsFAUbxcBGfRlvRd3ICYkRkCSgqKoexVX1ei+nXSRJQnR0tCG35tAKG81P9D6AjaJgoxjYqC8OszoKs1mREusA0LhFlxkoioLMzExD3s2oFTaan+h9ABtFwUYxsFFfHGZ1ZrZLDVRVhdvtNuTdjFpho/mJ3gewURRsFAMb9cVhVmf+m8BM8sosERERkZFwmNXZCN/2XIe5PRcRERFRoDjM6sx3mUFmiQs1bo/Oqzk+WZaRlJRkyLsZtcJG8xO9D2CjKNgoBjbqy3gr6mZ6R9jRO8IOVQX2FVTpvZzjkiQJ4eHhhrybUStsND/R+wA2ioKNYmCjvjjMGsAIE715gtfrRVpaGrxer95LCRo2mp/ofQAbRcFGMbBRXxxmDcB3qYFZbgIz4rYcWmOj+YneB7BRFGwUAxv1w2HWAHw7Gphley4iIiIio+AwawC+HQ325VfC4zXmTz1ERERERsRh1gD6x4QhzGZBvUdBdqlL7+UckyzLSElJMeTdjFpho/mJ3gewURRsFAMb9WW8FXVDsizhBBO9E5jVatV7CUHHRvMTvQ9goyjYKAY26ofDrEGYZUcDRVGQnp5u2IvAtcBG8xO9D2CjKNgoBjbqi8OsQZhtRwMiIiIiI+AwaxBH72igqqrOqyEiIiIyBw6zBjEkPgIWWUKZy43Cynq9l0NERERkCpLazV4GrKysRFRUFJxOJyIjI/VeThMz//0D9hdWYem1E/CHYfF6L6dVqqpCURTIsmzIt7TTAhvNT/Q+gI2iYKMY2Ki9QOY1vjJrIP5LDfKMfd2sx+PRewlBx0bzE70PYKMo2CgGNuqHw6yB+Hc0MPBNYIqiICsry5B3M2qFjeYneh/ARlGwUQxs1BeHWQPx7Wiw41AFvEq3uvqDiIiIqEM4zBrIuP49ER0WgoLKOny3r0jv5RAREREZHodZAwkNseCyickAgDc3Zuu7mGMw4lvZaY2N5id6H8BGUbBRDGzUD3czMJicshpM/9f3UFTg2/nTMDguQu8lEREREXUp7mZgYskxYTjjhMZtud7ceFDn1bSkqiqqq6uFfmMHNpqf6H0AG0XBRjGwUV8cZg3o2qkDAACfbM9FZV2DvotpRlEU5ObmGvJuRq2w0fxE7wPYKAo2ioGN+tJ9mF2yZAkGDBiA0NBQTJ48GVu2bDnm45999lkMHToUPXr0QHJyMubNm4e6urouWm3XmDqoF1LjwlHj9uLjrbl6L4eIiIjIsHQdZj/44APMnz8fCxYswPbt2zFmzBjMnDkTRUWt38n/7rvv4oEHHsCCBQuwd+9evPbaa/jggw/w0EMPdfHKg0uSJMw58ursW5uyoXCbLiIiIqJW6TrMLl68GDfeeCPmzp2L4cOH4+WXX0ZYWBiWLl3a6uM3btyIk08+GVdeeSUGDBiAs846C1dcccVxX801owtP7IuIUCuyS2uwPr1Y7+X4SZIEm80m7Nv1AWwUgeh9ABtFwUYxsFFfVr2+sNvtxrZt2/Dggw/6j8myjBkzZmDTpk2tfszUqVOxbNkybNmyBZMmTUJmZiZWrVqFq6++us2vU19fj/r6ev8/V1Y2vruW1+uF1+sF0HiCZFmGoihNLmxu67jvfYnbOu77vEcfB9DiOpO2jlssFoTZLLh4XF+8vvEg3vwpG9NTYyHLsv+9kY+3xmA29e/fH5IktVjL8ZraWrsRmo5eu6qq6N+/P1RVhdfrFaKptfOUkpICRVGafIzZm44+7juHAIRp8rFYLJAkqcnzVISm1tY+cOBAf6MoTc3PU//+/f2/L0rT0cd9jc2fq2Zuam3tAwcObPE91exNzY8f67mqdVPzxx+LbsNsSUkJvF4v4uPjmxyPj4/Hvn37Wv2YK6+8EiUlJTjllFOgqio8Hg/+/Oc/H/Myg0WLFmHhwoUtjmdkZCA8PBwAEBUVhcTERBQWFsLpdPofExsbi9jYWOTl5cHlcvmPJyQkIDo6GtnZ2XC73f7jSUlJCA8PR0ZGRpOTk5KSAqvVivT09CZrSE1NhcfjQVZWlv+YLMsYMmQIXC4XTk1Q8QaA9WnF+OnXNJx64jA4nU4UFBT4H+9wOJCcnIyysjKUlJT4jwerSVVV1NfXY9iwYQgJCQm4KTf3f9cA22w2DBw4UPem5ucpLS0N9fX1sNvtkCRJiKbm52nw4MEoKytDaWmp/6dsszcdfZ58z9OwsDBhmnx856m6uhoZGRn+56kITa2dp5iYGKiqisLCQmGajj5PvudqTEwM+vXrJ0RT8/NUWlqKvLw8/3NVhKbWzlOPHj1QW1srVNPR58n3XO3bty969+4d9KaMjAy0l277zB4+fBh9+/bFxo0bMWXKFP/x++67D+vXr8fmzZtbfMy6detw+eWX4x//+AcmT56MAwcO4K677sKNN96IRx55pNWv09ors76T5tu3zMg/JV7/5jasSyvG3JMHYMF5I3R/1cXr9eLAgQMYMmQILBaL7j8latHUfO0NDQ04cOAABg8eDIvFIkRT87Wrqor09HQMGjQIFotFiKajj/uep4MHD4bNZhOi6WgWiwUejwdpaWn+56kITc3XrigKMjIyMHjw4CZ/tGnmpubnyfdcTU1NRUhIiBBNzY83NDQgPT29xXPVzE3N1+J7rg4aNMj/dcze1Pz48Z6rWjdVVFQgJiamXfvM6vbKbGxsLCwWS5OftgGgsLAQCQkJrX7MI488gquvvho33HADAGDUqFFwuVy46aab8Ne//rXJE8jHbrfDbre3OG6xWJr8Txxo+50tAj3e/PN25LgkSbBYLLj25AFYl1aMj7fm4p6zhsJht7b6eK3W3p4m3xPTt8bjPd6nrce3dbwrm5ofl2W5xXPE7E1H8/1RX2v/HZi1qflx3zkExGk6mu9/NM3PodmbRDxPxzsuy7J/DaI0NV9La89Vsze1tUYtnqtGazr67zv6XNWiqS263QBms9kwfvx4rF271n9MURSsXbu2ySu1R6upqWnxL8kXq9MLzEE3LbU3UmIdqKr34N3Nh/ReDhEREZGh6Lqbwfz58/HKK6/gzTffxN69e3HLLbfA5XJh7ty5AIA5c+Y0uUHsvPPOw0svvYT3338fWVlZWLNmDR555BGcd955AU3wZiLLEm44NQUA8NTqfdiSVabreiRJgsPhaPJHfqJho/mJ3gewURRsFAMb9aXbNbM+L774Iv71r3+hoKAAY8eOxfPPP4/JkycDAE477TQMGDAAb7zxBgDA4/Hg8ccfx9tvv428vDz07t0b5513Hh5//HFER0e36+sF8l6/RqGqKm5/bwdW7spHjMOGFbedjOSYML2XRURERBQUgcxrug+zXc2MwywA1Lq9uOQ/G7EnrxLDEiLwyS1T4bB3/SXPiqKgrKwMMTExbV4XY3ZsND/R+wA2ioKNYmCj9gKZ18T8Ny6gHjYLXpkzAb0j7NhXUIW7P9gJPd4ZTFVVlJSUCHuNMsBGEYjeB7BRFGwUAxv1xWHWRBKjeuA/V4+HzSpjze+FeGbNfr2XRERERKQrDrMmM65fTzx54SgAwJLvM7BiZ57OKyIiIiLSD4dZE7pwXBJunj4QAHDvx7uQWVzdZV/b9+4tRrybUStsND/R+wA2ioKNYmCjvngDmEl5FRVzlm7GTwdKcemEJDx18Ri9l0RERESkCd4A1g1YZAnzzxwKAPhsRx4KnHVd8nUVRUF+fn6Lt58TCRvNT/Q+gI2iYKMY2KgvDrMmNr5/T0xKiUGDV8VrGzK75Guqqgqn02nIuxm1wkbzE70PYKMo2CgGNuqLw6zJ3TJ9EADg3c2H4Kxp0Hk1RERERF2Lw6zJnTa0N4YlRMDl9mLZ5oN6L4eIiIioS3GYNTlJkvDnI6/OLt2QhboGb9C/XmxsrCHvZtQKG81P9D6AjaJgoxjYqC/uZiAAj1fB9H+tQ15FLf4+eySuPqm/3ksiIiIi6jDuZtDNWC0ybjw1BQDwyg+Z8HiDd6ehoijIyckx5N2MWmGj+YneB7BRFGwUAxv1xWFWEJdN7IcYhw2Hymqwak9B0L6OqqpwuVyGvJtRK2w0P9H7ADaKgo1iYKO+OMwKoofNgmumDAAAvLwuw5BPNiIiIiKtcZgVyJwp/dEjxILf8yvxY3qJ3sshIiIiCjoOswLp6bDh8knJAICX12cE5WvIsoyEhATIsrhPHTaan+h9ABtFwUYxsFFfxlsRdcr1pzTeCLYxoxQVNW7NP78kSYiOjjbk1hxaYaP5id4HsFEUbBQDG/XFYVYwST3DMKi3AwDwS3a55p9fURRkZmYa8m5GrbDR/ETvA9goCjaKgY364jAroEkpvQAAW7JKNf/cqqrC7XYLfYMZG81P9D6AjaJgoxjYqC8OswKanBIDANiSVabzSoiIiIiCi8OsgCYeGWb3HK6Eq96j82qIiIiIgofDrID6RvdA3+ge8Coqth/S9rpZWZaRlJRkyLsZtcJG8xO9D2CjKNgoBjbqy3grIk0E61IDSZIQHh5uyLsZtcJG8xO9D2CjKNgoBjbqi8OsoCYdGWY3azzMer1epKWlwev1avp5jYSN5id6H8BGUbBRDGzUF4dZQfmum92ZU4F6j7ZPPCNuy6E1Npqf6H0AG0XBRjGwUT8cZgU1MNaB2HAb3B4Fu3Kdei+HiIiIKCg4zApKkiT/pQbcoouIiIhExWFWYJMGaH/drCzLSElJMeTdjFpho/mJ3gewURRsFAMb9WW8FZFmfNfNbssug8er3XUuVqtVs89lVGw0P9H7ADaKgo1iYKN+OMwKbFhCJCJCrXC5vdibX6XJ51QUBenp6Ya9CFwLbDQ/0fsANoqCjWJgo744zArMIkuY6L/UoFTn1RARERFpj8Os4HgTGBEREYmMw6zgfK/M/pJdBkVRdV4NERERkbYkVVW71YRTWVmJqKgoOJ1OREZG6r2coHN7FIxeuBp1DQrWzJuG1PiITn0+VVWhKApkWTbkW9ppgY3mJ3ofwEZRsFEMbNReIPMaX5kVnM0qY1y/ngC026LL4/Fo8nmMjI3mJ3ofwEZRsFEMbNQPh9luQMvrZhVFQVZWliHvZtQKG81P9D6AjaJgoxjYqC8Os92A780TtmSVoZtdVUJERESC4zDbDZzYryessoSCyjrklNXqvRwiIiIizXCY7QZ62CwYnRQFANiS3flLDYz4VnZaY6P5id4HsFEUbBQDG/XD3Qy6iSe/2oeX12dgxglxePWaiXovh4iIiKhN3M2AWrhgbB9YZAnf7i3Cqt35Hf48qqqiurpa6Gtv2Wh+ovcBbBQFG8XARn1xmO0mTkiMxC3TBwEAHlm+B6XV9R36PIqiIDc315B3M2qFjeYneh/ARlGwUQxs1BeH2W7kjjMGY2h8BEpdbjz6+W96L4eIiIio0zjMdiN2qwVPXzIGFlnCyl35nbrcgIiIiMgIOMx2M6OSojp1uYEkSbDZbMK+XR/ARhGI3gewURRsFAMb9cXdDLqheo8X57/wE/YXVmHW6EQsuXKc3ksiIiIi8uNuBnRMnbncQFVVVFRUGPJuRq2w0fxE7wPYKAo2ioGN+uIw202NSorCraf973KDMpe7XR+nKAoKCgoMeTejVthofqL3AWwUBRvFwEZ9cZjtxm7/w/92N3hvyyG9l0NEREQUMA6z3ZjdasH1p6YAAD7ZnmvIPzogIiIiOhYOs93cuaMSERoiI7PYhZ05Fcd9vCRJcDgchrybUStsND/R+wA2ioKNYmCjvribAeHu93dg+c7DuPqk/vj77JF6L4eIiIi6Oe5mQAG5aHwSAODzXw+j3uM95mMVRUFJSYkhLwDXChvNT/Q+gI2iYKMY2KgvDrOEqYNikRAZCmdtA77fV3TMx6qqipKSEqGvr2Wj+YneB7BRFGwUAxv1xWGWYJElzD6xLwDg4215Oq+GiIiIqP04zBIA4KJxjcPsuv1FAb/FLREREZFeOMwSACA1PgKjk6LgUVR8/uvhNh8nSRKioqIMeTejVthofqL3AWwUBRvFwEZ9cTcD8nvjpyz87YvfMbJvJL6841S9l0NERETdFHczoA45f2xfWGUJe/Iqsb+gqtXHKIqC/Px8Q97NqBU2mp/ofQAbRcFGMbBRXxxmyS/GYcPpw+IAAJ9uz231Maqqwul0GvJuRq2w0fxE7wPYKAo2ioGN+uIwS01cNK5xz9nPduTBqxjvCUtERER0NA6z1MTpw3ojOiwERVX12HCgRO/lEBERER0Th1lqwm614PwxfQAAn2xreamBJEmIjY015N2MWmGj+YneB7BRFGwUAxv1xd0MqIWdORWYveQn2K0yNj7wB/QKt+u9JCIiIupGuJsBdcqYpCiMTopCvUfBKz9mNfk9RVGQk5NjyLsZtcJG8xO9D2CjKNgoBjbqi8MstSBJEu78QyoA4K1N2Shzuf2/p6oqXC6XIe9m1AobzU/0PoCNomCjGNioLw6z1KozTojDiD6RqHF78dqGTL2XQ0RERNQqDrPUKkmScOcZja/OvrnxICpq3Mf5CCIiIqKux2GW2nTW8HickBiJ6noPXtvQeO2sLMtISEiALIv71GGj+YneB7BRFGwUAxv1ZbwVkWE0Xjs7GADwxk/ZcNY0QJIkREdHG3JrDq2w0fxE7wPYKAo2ioGN+uIwS8c0c0QChsZHoKreg6U/ZUFRFGRmZhrybkatsNH8RO8D2CgKNoqBjfriMEvHJMv/u3Z26U9ZcNa44Xa7DXk3o1ZUVWWjyYneB7BRFGwUAxv1xWGWjuuckQlIjQtHVZ0Hb246qPdyiIiIiPw4zNJxybKEO/yvzmbD5fbqvCIiIiKiRhxmqV1mjUrEoN4OVNZ5sCZHNeTdjFqRZRlJSUlsNDHR+wA2ioKNYmCjvoy3IjIkiyzhrhlDAAD/3XAQa/cW6byi4JEkCeHh4Ya8Y1MrojeK3gewURRsFAMb9cVhltrtvNGJuHR8EhQVuOO9HdiVW6H3koLC6/UiLS0NXq+4l1OI3ih6H8BGUbBRDGzUF4dZajdJkvDYBcMxrk8P1DZ4cd0bW5FTVqP3soLCiFuPaE30RtH7ADaKgo1iYKN+OMxSQEIsMv56WgKGJUSgpLoec9/4Bc6aBr2XRURERN0Uh1kKmMMm47U545EYFYoDRdW46e2tqPcY748diIiISHySasTdb4OosrISUVFRcDqdiIyM1Hs5puPbNNlms2FfQRUueXkTqus9uGBsH/z70rGQZeNdGB6ooxuNeKG7FkRvFL0PYKMo2CgGNmovkHmNr8xSwKxWKwDghMRIvHTVOFhlCSt2HsbDK/ZAUcT42cjXKDLRG0XvA9goCjaKgY364TBLAVEUBenp6f6LwE9N7Y2nLh4NSQLe3XwI93z0KzxeY14g3l7NG0UkeqPofQAbRcFGMbBRXxxmqdMuHJeE5y4/EVZZwqc78nD7uzt4DS0RERF1CQ6zpInzx/TBS1eNh80i4+vfCnDTW9tQy7e9JSIioiDjMEuaOXN4PJZeOxE9QixYn1aMa17fgqo6bttFREREwcPdDCggqqpCURTIstzm3Yxbs8sw9/VfUFXvwdjkaHxw80mwWy1dvNKOa0+j2YneKHofwEZRsFEMbNQedzOgoPJ4PMf8/QkDYvDujSchOiwEO3Mq8ObG7K5ZmIaO1ygC0RtF7wPYKAo2ioGN+uEwSwFRFAVZWVnHvZtxVFIU/nruCQCAF9YeQEl1fVcsTxPtbTQz0RtF7wPYKAo2ioGN+uIwS0Fz0bgkjOobhap6D575Zr/eyyEiIiIBcZiloJFlCY+eNxwA8P4vOfjtsFPnFREREZFoOMxSwGS5/U+biQNi8MfRiVBV4LEvfodZ7jcMpNGsRG8UvQ9goyjYKAY26oe7GVDQ5VXU4g9Pr0O9R8FLfxqHc0Yl6r0kIiIiMjDuZkBBo6oqqqurA3qFtW90D9w8bSAA4PFVe1HXYOw3U+hIo9mI3ih6H8BGUbBRDGzUF4dZCoiiKMjNzQ34bsabpw9CfKQdueW1WPpTVpBWp42ONpqJ6I2i9wFsFAUbxcBGfXGYpS7hsFtx/9nDAABLvjuAoso6nVdEREREIuAwS11m9ti+GJMcDZfbiwWf/wavYrw/qiAiIiJz4TBLAZEkCTabrUNvZSfLEv523nDIEvDVngLM/3AnPF7j/XFFZxrNQvRG0fsANoqCjWJgo764mwF1uZW78nHX+zvgUVScMzIBz11+ImxW/lxFREREjbibAQWNqqqoqKjo1N2Ms0Yn4qWrxsNmkfHVngLcsmyboXY40KLR6ERvFL0PYKMo2CgGNuqLwywFRFEUFBQUdPpuxjOHx+OVaybAbpWxdl8RbnxrK2rdxhhotWo0MtEbRe8D2CgKNoqBjfriMEu6mT6kN16fOxFhNgt+TC/Bta9vgaveo/eyiIiIyEQ4zJKupg6KxVvXTUK43YrNWWX41+r9ei+JiIiITITDLAVEkiQ4HA5N72acMCAGL155IgDg3c2HkO+s1exzd0QwGo1G9EbR+wA2ioKNYmCjvribARmCqqq47L8/Y0tWGa4+qT/+Pnuk3ksiIiIinXA3AwoaRVFQUlKi+QXgkiRh/plDAADv/3IIeRX6vTobrEYjEb1R9D6AjaJgoxjYqC8OsxQQVVVRUlISlK05ThrYC1MG9kKDV8WL3x3Q/PO3VzAbjUL0RtH7ADaKgo1iYKO+OMySocw78ursR1tzkFNWo/NqiIiIyOg4zJKhTEqJwampsfAo+r46S0RERObAYZYCIkkSoqKigno3490zGl+d/Xh7Lg6WuoL2ddrSFY16E71R9D6AjaJgoxjYqC/uZkCGdM3SLVifVoyLxyfh6UvG6L0cIiIi6kLczYCCRlEU5OfnB/1uRt+1s59uz0VWSde+OttVjXoSvVH0PoCNomCjGNioL92H2SVLlmDAgAEIDQ3F5MmTsWXLlmM+vqKiArfddhsSExNht9sxZMgQrFq1qotWS6qqwul0Bv1uxrHJ0ThjWBwUFXhhbXpQv1ZzXdWoJ9EbRe8D2CgKNoqBjfrSdZj94IMPMH/+fCxYsADbt2/HmDFjMHPmTBQVFbX6eLfbjTPPPBPZ2dn4+OOPsX//frzyyivo27dvF6+cuoLv2tnlO/Ow7WC5zqshIiIiI9J1mF28eDFuvPFGzJ07F8OHD8fLL7+MsLAwLF26tNXHL126FGVlZVi+fDlOPvlkDBgwANOnT8eYMbymUkSjkqIwe2wfKCpwy7JtKKqs03tJREREZDBWvb6w2+3Gtm3b8OCDD/qPybKMGTNmYNOmTa1+zOeff44pU6bgtttuw4oVK9C7d29ceeWVuP/++2GxWFr9mPr6etTX1/v/ubKyEgDg9Xrh9XoBNN6hJ8syFEVp8vJ5W8dlWYYkSW0e933eo48DaHGdSVvHLRYLVFVtcty3lraOt3ftnW1SFAUxMTEA0GItwWhaeP5w/Ha4EulF1bjlne1478aTYJUR9PMUExPjX5MZz1NrTc316tVLqKajjx99DkVpOprv+93Rz1MRmpqvXVVVxMbGAkCTz2/mpubnyfdc9X19EZpaO97ac9XMTc3X4nuuqqrarueqGZqaHz/ec1XrpuaPPxbdhtmSkhJ4vV7Ex8c3OR4fH499+/a1+jGZmZn47rvv8Kc//QmrVq3CgQMHcOutt6KhoQELFixo9WMWLVqEhQsXtjiekZGB8PBwAEBUVBQSExNRWFgIp9Ppf0xsbCxiY2ORl5cHl+t/NyElJCQgOjoa2dnZcLvd/uNJSUkIDw9HRkZGk5OTkpICq9WK9PSm136mpqbC4/EgKyvLf0yWZQwZMgQulwu5ubn+4zabDQMHDoTT6URBQYH/uMPhQHJyMsrKylBSUuI/HuymqKioLmt64JSeuPOLGmw7WI6/f/k7bp3cK6jnKSMjAwBQVlZm+vN0rOdeZGSkv1WUpubnqaKiQrgm33mqra1FWVmZ/3kqQlNb56miokK4pubnqb6+Xrgm33mqqKho8lwVoamt85Sfny9cU/PzJMtylzQd/f+n4+nw1lzFxcXYv38/AGDo0KHo3bt3QB9/+PBh9O3bFxs3bsSUKVP8x++77z6sX78emzdvbvExQ4YMQV1dHbKysvyvTCxevBj/+te/kJ+f3+rXae2VWd9J8231IOJPicF8Zfbw4cNISkryf56uaPp+XxFuXLYdqgr886JRuHhc32M+PpCm5mv3eDw4fPgw+vTpA1mWTXmemje19spsXl4eEhMT/Y8xe1PzV2Z95zAkJESIpqNZLBZ4vV7k5ub6n6ciNLX2ymx+fj769OnT5LFmbmrtlVnf/w+tVqsQTc2Pezwe5OXltXiumrmptVdm8/PzkZiY2GQfVjM3tfbK7LGeq1o3VVRUICYmpl1bcwX8yqzL5cIdd9yBt99+278wi8WCOXPm4IUXXkBYWFi7Pk9sbCwsFgsKCwubHC8sLERCQkKrH5OYmIiQkJAmlxSccMIJKCgogNvths1ma/Exdrsddru9xXGLxdLi0oSj/6femeNtXfIQyHFJkgI6rtXa29NUW1t7zLU0f7xPZ5pmjEjE3WcMwb+/TcMjK37DsIRIjEmObtfaAz1PsiyjtrYWsiw3eYzZztOxjnu9XtTU1LRoBMzb1Py47xwC4jQ119rz1MxNzdfu9Xr9r/yI0tTa8draWv8AJEpT8+OtPVfN3NR8Lb7nqlb//RmhqbXjnXmuavV9r9Wv2e5HHjF//nysX78en3/+OSoqKlBRUYEVK1Zg/fr1+Mtf/tLuz2Oz2TB+/HisXbvWf0xRFKxdu7bJK7VHO/nkk3HgwIEmU3xaWhoSExNbHWRJLHf8YTBmnBAPt0fBn5dtQ0l1/fE/iIiIiIQW8DD7ySef4LXXXsM555yDyMhIREZG4txzz8Urr7yCjz/+OKDPNX/+fLzyyit48803sXfvXtxyyy1wuVyYO3cuAGDOnDlNbhC75ZZbUFZWhrvuugtpaWlYuXIlnnjiCdx2222BZpAJybKExZeNwcBYB/Kddbj29S3ILK7We1lERESko4AvM6ipqWlx0xYAxMXFoaamJqDPddlll6G4uBiPPvooCgoKMHbsWHz99df+z3/o0KEmL1cnJydj9erVmDdvHkaPHo2+ffvirrvuwv333x9oBnWQLMtISEho848Rgi0yNAT/nTMeF720CXvyKnHu8z/i/rOH4ZopAyDL2rxftN6NXUH0RtH7ADaKgo1iYKO+Ar4B7IwzzkCvXr3w1ltvITQ0FEDjNRTXXHMNysrK8O233wZloVoJ5L1+ybgOV9Tivo93YcOBxjsvpwzshacuHo3kmPZds01ERETGFci8FvB4/eyzz+Knn35CUlISzjjjDJxxxhlITk7Gxo0b8dxzz3V40WQOiqIgMzOz1bvju1Kf6B54+/pJ+PvskegRYsGmzFKc89yP+OCXQwjw57MWjNIYTKI3it4HsFEUbBQDG/UV8GUGo0aNQnp6Ot555x3/frBXXHEF/vSnP6FHjx6aL5CMRVVVuN3uTg+MWpAkCVef1B+nDo7FvR//il+yy3H/J7ux7WA5nrxwdIcvOzBSY7CI3ih6H8BGUbBRDGzUV8DD7A8//ICpU6fixhtvbHLc4/Hghx9+wLRp0zRbHFF7DIh14P2bpuC1DZl48qt9+HBrLrwK8NTFo2HR6DpaIiIiMqaALzM4/fTT/e/icTSn04nTTz9dk0URBcoiS7hp2iA8d/mJsMgSPtmei3s/+hVexXg/QRIREZF2Ah5mVVVt8u4WPqWlpXA4HJosioxLlmX/u38Z0Xlj+uD5IwPtpzvyMP/DnfB4A7u+x+iNWhC9UfQ+gI2iYKMY2Kivdl9mcOGFFwJovE7x2muvbfKuWl6vF7t27cLUqVO1XyEZiiRJCA8P13sZxzRrdCJkCbjjvR1YsfMwFBX496VjYLW07z9AMzR2luiNovcBbBQFG8XARn21e7yOiopCVFQUVFVFRESE/5+joqKQkJCAm266CcuWLQvmWskAvF4v0tLSWrzHstGcMyoRL145DlZZwhe/HsZdH7T/FVqzNHaG6I2i9wFsFAUbxcBGfbX7ldnXX38dADBgwADcc889vKSgGzPithytOXtkAl66ajxufWcbVu7Kx4BeYbh35rB2faxZGjtD9EbR+wA2ioKNYmCjfgK+8GHBggVwOBwoLi7Ghg0bsGHDBhQXFwdjbUSddubweDx9yRgAwJLvM7A+jc9VIiIikQQ8zNbU1OC6665DYmIipk2bhmnTpqFPnz64/vrrA347W6KucMHYvvjT5H4AgHkf7ES+s1bnFREREZFWAh5m582bh/Xr1+OLL75ARUUFKioqsGLFCqxfvx5/+ctfgrFGMhBZlpGSkmLIuxmP5ZE/DsfwxEiUudy4870dx7x+1qyNgRC9UfQ+gI2iYKMY2KgvSQ3wrRxiY2Px8ccf47TTTmty/Pvvv8ell15q+EsOAnmvX2pJVVUoigJZllvdos3Isktc+OMLG1Bd78Etpw3C/We3fv2smRvbS/RG0fsANoqCjWJgo/YCmdc6dJlBfHx8i+NxcXG8zKAbUBQF6enphr0I/FgGxDrw5EWjAAAvrcvA9/uLWn2cmRvbS/RG0fsANoqCjWJgo74CHmanTJmCBQsWoK6uzn+strYWCxcuxJQpUzRdHJHW/ji6D64+qT8AYP4HO3G4gtfPEhERmVm7t+byee655zBz5kwkJSVhzJjGu8R//fVXhIaGYvXq1ZovkEhrf511AnbklGNPXiVueWc73r5+EiJDQ/ReFhEREXVAwK/Mjhw5Eunp6Vi0aBHGjh2LsWPH4sknn0R6ejpGjBgRjDUSaSo0xIIlV45DZKgVv+ZU4LL//IyiqrrjfyAREREZTsA3gJkdbwDrHJEuct+T58S1r29BSbUb/XuF4e3rJqNfrzChGtsieqPofQAbRcFGMbBRe4HMax0aZg8fPowNGzagqKioxYXAd955Z6CfrktxmO0cVVXhdrths9mE+A82u8SFq17bjNzyWvSOsOOt6yZhWEKEUI2tEe08Nid6H8BGUbBRDGzUXlCH2TfeeAM333wzbDYbevXq1SRIkiRkZmZ2bNVdhMNs53i9XqSnpyM1NRUWi0Xv5WiisLIOc17bgv2FVYgMteKVOeMR5S4RqrE5Ec/j0UTvA9goCjaKgY3aC+rWXI888ggeffRROJ1OZGdnIysry//L6IMsUWviI0Px4c1TML5/T1TWeXDN679gS65L72URERFRO3Ron9nLL7/ckO8AQdRRUWEhWHb9ZJw+tDfqGhQ89l0BvtpToPeyiIiI6DgCnkivv/56fPTRR8FYC5mEqD/I9LBZ8N85E/DH0QnwKMCd7+/Ex9ty9V5W0Ih6Hn1E7wPYKAo2ioGN+gn4mlmv14s//vGPqK2txahRoxAS0nR/zsWLF2u6QK3xmlk6Hq+i4qFPd+ODrTkAgL9fMAJXTxmg76KIiIi6kUDmtYDfNGHRokVYvXo1hg4dCgAtbgAjsamqCpfLBYfDIez5liXgrzNT0MNmwRsbs/HIit9Q4/bi5umD9F6aZkQ/j6L3AWwUBRvFwEZ9Bfx68TPPPIOlS5di7969WLduHb7//nv/r++++y4YayQDURQFubm5hnxvZq0oioLDeXl4+NyhuP30wQCARV/tw+I1aRBlW2bRz6PofQAbRcFGMbBRXwEPs3a7HSeffHIw1kJkKJIk4Z6ZQ3Hf2Y1/CvH82nT8Y+VeYQZaIiIiEQQ8zN5111144YUXgrEWIkO69bTBWHh+41s1v7YhCw99tgdehQMtERGREQR8zeyWLVvw3Xff4csvv8SIESNa3AD26aefarY4Mh5JkoR+hxOg9cZrpg5AD5sFD3yyC+9tOYRatwdPXzIGVosx7+w8HtHPo+h9ABtFwUYxsFFfAe9mMHfu3GP+/uuvv96pBQUbdzOgzvji18OY98FOeBQVM0fE4/krToTdKua7vRAREeklqG9na3YcZjtHVVU4nU5ERUUZ8qczLRyv8dvfC3HrO9vh9iqYNqQ3/nPVePSwmWugFf08it4HsFEUbBQDG7UX1Lezpe5NURQUFBQY8m5GrRyvccbweCy9diJ6hFjwQ1ox5r6xBfUebxevsnNEP4+i9wFsFAUbxcBGfbXrmtlx48Zh7dq16NmzJ0488cRjTuTbt2/XbHFERnVKaizevn4S5r7+C37OLMNDn+7B05eMFvYnciIiIqNq1zB7wQUXwG63AwBmz54dzPUQmcaEATH4f1eNw7Wv/4JPtudiWEIEbpw2UO9lERERdSvtGmYXLFjQ6t9T9yNJkiHf/UNLgTSemtobj8w6AX/74nc88dVeDI4Lx+nD4rpglZ0j+nkUvQ9goyjYKAY26qvDN4C53W4UFRW1uHaiX79+miwsWHgDGGlNVVU89NkevLflEMLtVnx261SkxkfovSwiIiLTCuoNYGlpaTj11FPRo0cP9O/fHykpKUhJScGAAQOQkpLS4UWTOSiKgpKSEkNeAK6VQBslScLC80dgUkoMqus9uOGtrSh3uYO8ys4R/TyK3gewURRsFAMb9RXwMDt37lzIsowvv/wS27Ztw/bt27F9+3bs2LGDN391A6qqoqSkROi3dO1Io80q4+WrxiOpZw8cLK3Bre9sR4PXeP/B+4h+HkXvA9goCjaKgY36CvgdwHbu3Ilt27Zh2LBhwVgPkWnFOGx47ZqJuPD//YRNmaW496Nf8dTFY2Czcgc8IiKiYAn4/7LDhw9HSUlJMNZCZHpDEyLw/BUnwiJLWL7zMK56dTPKDH7JARERkZkFPMz+85//xH333Yd169ahtLQUlZWVTX6R2CRJEvodToDON55xQuObKkTYrdiSXYbZS37CgaIqjVfZOaKfR9H7ADaKgo1iYKO+At7NQJYb59/mMaqqQpIkeL3Gfick7mZAXSW9sArXv7kVh8pqEBFqxZIrx2HakN56L4uIiMjwApnXAr5m9vvvv+/wwsj8FEVBYWEh4uPj/T/YiEarxtT4CCy/7WT8+e1t2JJdhrlv/IIF5w3HnCkDtFtsB4l+HkXvA9goCjaKgY36CniYnT59ejDWQSahqiqcTifi4oz/xgAdpWVjjMOGt2+YhIc+3YNPtufi0RW/YVNGKRaePwJxkaEarLZjRD+PovcBbBQFG8XARn0FPMz61NTU4NChQ3C7m97cMnr06E4vikgkdqsFT18yGqnx4Xh69X58tacAPx0owV9nnYBLJyQb8vojIiIiswh4mC0uLsbcuXPx1Vdftfr7Rr9mlkgPkiThz9MHYVpqbzzw6S7synXi/k92Y/mOw1h04SgMiHXovUQiIiJTCviih7vvvhsVFRXYvHkzevToga+//hpvvvkmUlNT8fnnnwdjjWQgkiQhNjZW6FcTg9k4vE8kPr1lKh6edQJCQ2RsyizFzGd/wGsbsjT/Wsci+nkUvQ9goyjYKAY26ivg3QwSExOxYsUKTJo0CZGRkdi6dSuGDBmCzz//HE899RQ2bNgQrLVqgrsZkFEcKq3Bg5/twk8HSgEAz10+FheM7avzqoiIiPQXyLwW8CuzLpfLf/Fvz549UVxcDAAYNWoU3862G1AUBTk5OYZ8b2atdFVjv15hWHb9ZNx62iAAwMPL9yC3vCaoX9NH9PMoeh/ARlGwUQxs1Fe7h9lDhw5BURQMHToU+/fvBwCMGTMG//nPf5CXl4eXX34ZiYmJQVsoGYOqqnC5XIZ8b2atdGWjJEmYf+YQjOsXjao6D/7y4a/wKsH/uqKfR9H7ADaKgo1iYKO+2j3MpqSkoKSkBHfddRfy8/MBAAsWLMBXX32Ffv364fnnn8cTTzwRtIUSicpqkfHvy8bCYbNgc1YZ/vtDpt5LIiIiMo1272bgm8Svuuoq/7Hx48fj4MGD2LdvH/r164fY2FjtV0jUDfTv5cCC80fgvo93YfGa/Tg1NRYj+0bpvSwiIiLDC+ia2dbuYAsLC8O4ceM4yHYTsiwjISHBcO/+oSW9Gi8Zn4SzRySgwavirvd3oNYdvG3uRD+PovcBbBQFG8XARn21ezcDWZZx0003ISws7JiPW7x4sSYLCxbuZkBGVuZy4+xnf0BRVT2umdIfCy8YqfeSiIiIulzQdjPYvXs3duzY0eavnTt3dmbdZAKKoiAzM9OQdzNqRc/GGIcNT18yBgDw5qaD+H5/UVC+jujnUfQ+gI2iYKMY2KivgN4B7LPPPjPke/JS11FVFW6325B3M2pF78ZpQ3pj7skD8PpP2bjrvR346M9TMTQhQtOvoXdjsIneB7BRFGwUAxv11e5XZo34jg9Eorr/7GEY1y8alXUezFm6ucv2nyUiIjKbdg+zRpzEiUQVGmLB0msnIjUuHIWV9Zjz2haUVtfrvSwiIiLDafcw+/rrryMqilsFdXeyLCMpKcmQdzNqxSiN0WE2vHX9JPSN7oHMEhfmvvELqus9mnxuozQGi+h9ABtFwUYxsFFf7d7NQBTczYDMJqO4Gpe8vAllLjdOHtwLS6+dCLvVoveyiIiIgiZouxkQeb1epKWlwesN3h6oejNa46De4Xhj7kQ4bBb8dKAU8z/o/FveGq1Ra6L3AWwUBRvFwEZ9cZilgBlxWw6tGa1xdFI0/nP1BIRYJKzcnY9pT32PZ79NQ15FbYc/p9EatSZ6H8BGUbBRDGzUD4dZIpM4JTUWL1wxDlE9QpBXUYtnv03HKf/8Dtcs3YKvdufD7THmNxkiIqJgCmifWZ+Kigp8/PHHyMjIwL333ouYmBhs374d8fHx6Nu3r9ZrJKIjzh6ZgNOG9sbq3wrw/pYcbMosxfq0YqxPK0Z8pB3/vnQspg7mW0sTEVH3EfANYLt27cKMGTMQFRWF7Oxs7N+/HwMHDsTDDz+MQ4cO4a233grWWjXBG8A6x7dpss1mE3bvYTM1Hix14cOtOfhoay6KquohS8D8M4fg1tMGQ5bbXruZGjtC9D6AjaJgoxjYqL2g3gA2f/58XHvttUhPT0doaKj/+Lnnnosffvgh8NWS6VitHXpB31TM0ti/lwP3zhyG9feejkvGJ0FRgae/ScN1b/6Ccpf7mB9rlsaOEr0PYKMo2CgGNuon4GH2l19+wc0339zieN++fVFQUKDJosi4FEVBenq6YS8C14IZG3vYLPjXJWPw1EWjYbfKWLe/GH98YQN25lS0+ngzNgZC9D6AjaJgoxjYqK+Ah1m73Y7KysoWx9PS0tC7d29NFkVEHXPpxGR8duvJGNArDHkVtbjk5Y34f+sOoMatzZstEBERGU3Aw+z555+Pxx57DA0NDQAASZJw6NAh3H///bjooos0XyARBWZ4n0h8fscpOGdkAhq8Kp76ej9OfvI7vLA2Hc7aBr2XR0REpKmAh9lnnnkG1dXViIuLQ21tLaZPn47BgwcjIiICjz/+eDDWSEQBigwNwf/70zj86+LR6N8rDOU1DXhmTRpOefI7PPX1PpQe53paIiIis+jw29lu2LABu3btQnV1NcaNG4cZM2Zovbag4G4GnaOqKhRFgSzLQt+xKVKjx6tg5e58LPn+ANIKqwEAoSEybj51IG45fRBCQ4x5QX9niHYOW8NGMbBRDGzUXiDzWoeHWbPiMNs53H7EvBRFxbd7C7Hk+wP4NdcJABjY24En/m8UThrYS+fVaUvUc3g0NoqBjWJgo/aCOsw+//zzrX8iSUJoaCgGDx6MadOmwWKxBPJpuwyH2c7xer1IT09HamqqYc9xZ4neqKoqvvg1DwtW7EF5beN7bF8yPgkPnXsCejpsOq9OG6KfQ4CNomCjGNiovUDmtYD/fPHf//43iouLUVNTg549ewIAysvLERYWhvDwcBQVFWHgwIH4/vvvkZyc3LECIgoaSZIwa1Qi+khOfJrhxbtbcvDRtlys3VeEh2edgP87sa+wrywQEZF4Ar4B7IknnsDEiRORnp6O0tJSlJaWIi0tDZMnT8Zzzz2HQ4cOISEhAfPmzQvGeolII+F2C/5+wQh8cssUDIkPR5nLjfkf/orr39yKAmed3ssjIiJql4CH2Ycffhj//ve/MWjQIP+xwYMH4+mnn8aDDz6IpKQkPPXUU/jpp580XSgZhywH/LQxne7UOL5/DL6841TcO3MobBYZ3+0rwpn/Xo+Pt+XCzJfUd6dzKDI2ioGNYjBqY8DXzIaFheGHH37AhAkTmhz/5ZdfMH36dNTU1CA7OxsjR45EdXW1povVAq+ZJWpbWmEV7v3oV/8NYn8YFodFF45CfGTocT6SiIhIO4HMawGP2Keffjpuvvlm7Nixw39sx44duOWWW/CHP/wBALB7926kpKQE+qnJBFRVRXV1talfsTue7tw4JD4Cn9wyFfedfdSrtIvX461N2aisM88bLnTncygSNoqBjWIwcmPAw+xrr72GmJgYjB8/Hna7HXa7HRMmTEBMTAxee+01AEB4eDieeeYZzRdL+lMUBbm5uYZ8b2atdPdGq0XGracNxpd3noLRSVGorPPg0RW/YeI/vsUd7+3Auv1F8HiN/e+mu59DUbBRDGwUg5EbA97NICEhAWvWrMG+ffuQlpYGABg6dCiGDh3qf8zpp5+u3QqJSBdD4iPw6S1T8fbPB/Hu5kNIL6rGF78exhe/HkbvCDtmj+2D608ZiIQoXoJARET66fBb/wwbNgzDhg3Tci1EZDBWi4y5J6fg2qkDsDvPiU+352HFzjwUV9XjlR+z8PG2XCy+bCxOHxqn91KJiKib6tAwm5ubi88//xyHDh2C2930Pd4XL16sycLImCRJEvodTgA2tvX40UnRGJ0UjYfOPQHr9hfh+e/SsSevEnNf/wW3nT4I82YMgdVijDtdeQ7FwEYxsFEMRm4MeDeDtWvX4vzzz8fAgQOxb98+jBw5EtnZ2VBVFePGjcN3330XrLVqgrsZEGmj3uPF4yv34q1NBwEAk1Ni8MIVJyKOOx8QEVEnBXU3gwcffBD33HMPdu/ejdDQUHzyySfIycnB9OnTcckll3R40WQOqqqioqLCkHczaoWN7WO3WvDYBSPxwhUnwmGzYHNWGc59/kdsPFCi4Uo7hudQDGwUAxvFYOTGgIfZvXv3Ys6cOQAAq9WK2tpahIeH47HHHsM///lPzRdIxqIoCgoKCgx5N6NW2BiY88b0wRd3nIJhCREoqXbjT69txmNf/A5XvUeDlXYMz6EY2CgGNorByI0BD7MOh8N/nWxiYiIyMjL8v1dSov8rMkTU9Qb2Dsfy207G5ROToarA0p+ycNa/f8C6/UV6L42IiAQX8DB70kknYcOGDQCAc889F3/5y1/w+OOP47rrrsNJJ52k+QKJyBxCQyx48qLReGPuRPSN7oG8ilpc+/ovuPv9HSitrtd7eUREJKiAh9nFixdj8uTJAICFCxfijDPOwAcffIABAwb43zSBxCVJEhwOhyHvZtQKGzvntKFx+GbeNFx/SgpkCVi+8zBmLF6Pj7bmwKt0zbVWPIdiYKMY2CgGIzcGtJuB1+vFTz/9hNGjRyM6OjqIywoe7mZA1HV+zanA/Z/swr6CKgDAwFgHbjt9MC4Y28cw23gREZHxBG03A4vFgrPOOgvl5eWdWiCZl6IoKCkpMeQF4Fpho3bGJEfjiztOwUPnDkPPsBBklrjwl49+9b9S2xCkt8XlORQDG8XARjEYuTHgl0ZGjhyJzMzMYKyFTEBVVZSUlBhyaw6tsFFbIRYZN00bhB/v/wPuP3sYYhw2ZJfW4N6Pd+GMZ9Zj1e58zb8mz6EY2CgGNorByI0BD7P/+Mc/cM899+DLL79Efn4+Kisrm/wiImpNuN2KW04bhA33n46Hzh2G2HAbDpXV4NZ3tuOej35FtY5beRERkXkF/Ha25557LgDg/PPPb3IRsKqqkCQJXq9Xu9URkXDCbFbcNG0Qrj5pAF78Ph3/b10GPt6Wi1+yy/Dc5SdibHK03kskIiITCXiY/f7774OxDjIJSZIQFRVlyLsZtcLGrtHDZsG9M4dhWmpvzPtgJw6W1uCilzZi3oxU3HLaYFjkjq/NCH3BxkYxsFEMbNRXQLsZiIC7GRAZj7OmAQ8t342Vuxqvn52UEoP7zx6Kcf16GvIbJxERBVfQdjPw+fHHH3HVVVdh6tSpyMvLAwC8/fbb/jdTIHEpioL8/HxD3s2oFTZ2vaiwELx4xYl4+pIxcNgs2JJVhote2oRznvsRb2/KRlVdQ0Cfz2h9wcBGMbBRDGzUV8DD7CeffIKZM2eiR48e2L59O+rrG9/Zx+l04oknntB8gWQsqqrC6XQa8m5GrbBRH5Ik4eLxSVh116m4eHwSQkNk7CuowiMrfsPkJ9biwU934bfDznZ9LiP2aY2NYmCjGNiorw7tZvDyyy/jlVdeQUhIiP/4ySefjO3bt2u6OCLqfvr3cuDpS8Zg84MzsOC84RgcF44atxfvbcnBrOc34LZ3tyOrxKX3MomIyCACHmb379+PadOmtTgeFRWFiooKLdZERISosBDMPTkFa+ZNwwc3nYRZoxMhScDKXfk4c/F6PLx8N4qq6vReJhER6SzgYTYhIQEHDhxocXzDhg0YOHCgJosi45IkCbGxsULflMNGY5EkCZMH9sKSK8dh1Z2n4g/D4uBRVCz7+RCmP7UOz3yzH5XNrqk1U19HsVEMbBQDG/UV8G4GixYtwrJly7B06VKceeaZWLVqFQ4ePIh58+bhkUcewR133BGstWqCuxkQmd/mzFI8+fU+7DhUAQDoEWLBeWMScfmkfjgxOdqQ32yJiKj9ApnXAh5mVVXFE088gUWLFqGmpgYAYLfbcc899+Dvf/97x1fdRTjMdo6iKMjLy0Pfvn0hyx3aDMPw2GgOqqpi9W+FWLxmP9IKq/3HhyVE4LIJSZgcL2HYoP6m7TseEc7h8bBRDGwUQ1c3BjKvBfymCZIk4a9//SvuvfdeHDhwANXV1Rg+fDjCw8M7vGAyD1VV4XK5DHk3o1bYaA6SJOHskQmYOSIeWw+W473Nh7Bydz72FVRh4Zd7YbNIOH1YKWaN7oMzhsXBYQ/4252hiXAOj4eNYmCjGIzcGPB392XLluHCCy9EWFgYhg8fHow1ERG1myRJmDggBhMHxGDBeSOwfGce3ttyCPsKqrD6t0Ks/q0QdquM04b2xrmjEnHGCfEIF2ywJSLqzgJ+nXjevHmIi4vDlVdeiVWrVsHr9QZjXUREAYsKC8E1Uwfgy9un4sXzknDL9IEY0CsM9R4Fq38rxF3v78Tkx7/F4jVpAb8RAxERGVPA18x6PB58/fXXeO+997BixQqEhYXhkksuwZ/+9CdMnTo1WOvUDK+Z7RzfpslGfX9mLbDR/I7uA4C9+VVYtTsfK3fn+/eo7RkWgttOH4yrTuqP0BCLnsvtENHPIcBGUbBRDF3dGNQbwI5WU1ODzz77DO+++y6+/fZbJCUlISMjo6OfrktwmCXqvhpvGivAU6v3I7O4cajtExWKu2cMwYXj+sJqEfPGDSIiswlkXuvUd+6wsDDMnDkT55xzDlJTU5Gdnd2ZT0cmoCgKMjMzDfnezFpho/m11dd401givrl7Gp66aDQSo0Jx2FmH+z7ZhVOf+h5///J3bD9UbsgbHJoT/RwCbBQFG8Vg5MYO3QXhe0X2nXfewdq1a5GcnIwrrrgCH3/8sdbrI4NRVRVut9sU/7PvKDaa3/H6rBYZl05Mxvlj+2DZzwex5PsDyHfW4bUNWXhtQxb6RIXi3FGJmDU6EWMNum+t6OcQYKMo2CgGIzcGPMxefvnl+PLLLxEWFoZLL70UjzzyCKZMmRKMtRERBVVoiAU3nDoQV53UHz+kFWPV7nys+b0Qh511eHVDFl7dkIUBvcJw8fgk/N+4JPSN7qH3komIqJmAh1mLxYIPP/wQM2fOhMXS9KaJPXv2YOTIkZotjoioK4SGWHDWiAScNSIBdQ1erD9qsM0urcHT36ThmTVpmDqoFy4en4SzRySih818N40REYmoUzeAAUBVVRXee+89vPrqq9i2bZvht+riDWCd49s02eFwGPKPXrXARvPTqs9V78HXewrw8bZcbMos9R8Pt1tx4bi+uOqk/hgSH6HFkgMm+jkE2CgKNoqhqxu7ZDeDH374Aa+99ho++eQT9OnTBxdeeCEuuugiTJw4sUOL7iocZomoI3LKavDp9jx8vD0HOWW1/uOTUmJw9Un9MXNEAmxW7oZARKSFoO1mUFBQgCeffBKpqam45JJLEBkZifr6eixfvhxPPvmk4QdZ6jyv14u0tDTDvwLfGWw0v2D0JceE4a4ZqVh/z+lYdv1knD0iARZZwpasMtzx3g5MffI7/O3z37Bqdz6KKus0+7ptEf0cAmwUBRvFYOTGdg+z5513HoYOHYpdu3bh2WefxeHDh/HCCy9osoglS5ZgwIABCA0NxeTJk7Fly5Z2fdz7778PSZIwe/ZsTdZB7WPEbTm0xkbzC1afLEs4JTUWL189Hj/d/wfcdUYq4iLsKKmuxxsbs3HrO9sx6Ym1OPWp7zDvg51Y9vNBHK6oPf4n7gDRzyHARlGwUQxGbWz3DWBfffUV7rzzTtxyyy1ITU3VbAEffPAB5s+fj5dffhmTJ0/Gs88+i5kzZ2L//v2Ii4tr8+Oys7Nxzz334NRTT9VsLUREgUiICsW8M4fg9j8Mxnf7irAhvQRbD5ZjX0ElcspqkVOWh8925AFovBxh9ti+mDUqEVFhITqvnIhIHO0eZjds2IDXXnsN48ePxwknnICrr74al19+eacXsHjxYtx4442YO3cuAODll1/GypUrsXTpUjzwwAOtfozX68Wf/vQnLFy4ED/++CMqKira/Pz19fWor6/3/3NlZaX/c/heKpckCbIsQ1GUJvuntXVclmVIktTm8eYvwcty4wvgzX+iaeu4xWKBqqpNjvvW0tbx9q69s01er9f/mOZrMWtT87X7Gn2/L0JT87X7zl/zx5u56ejjR5/DrmiySMCMYb0xY1hvAIDL7cXOHCd+yS7Dz5ml+CW7HFuyyrAlqwwLPt+D04b0xgVj++C0Ib3Rw2bp1PeI5t/HzHSe2mry8f198+eqmZuanyffc1VRFFgsFiGaWltja89VszcdvRbfY1p7oxazNjU/frznqtZNgVzO0O5h9qSTTsJJJ52EZ599Fh988AGWLl2K+fPnQ1EUrFmzBsnJyYiICOyuXrfbjW3btuHBBx/0H5NlGTNmzMCmTZva/LjHHnsMcXFxuP766/Hjjz8e82ssWrQICxcubHE8IyMD4eHhAICoqCgkJiaisLAQTqfT/5jY2FjExsYiLy8PLpfLfzwhIQHR0dHIzs6G2+32H09KSkJ4eDgyMjKanJyUlBRYrVakp6c3WUNqaio8Hg+ysrKa9A8ZMgQulwu5ubn+4zabDQMHDoTT6URBQYH/uMPhQHJyMsrKylBSUuI/Hqwm338EHo8HkiQJ0dT8PPmOZ2RkQJIkIZqan6fBgwejb9++/kYRmo4+T77naXZ2tm5N04b0RpK1CrP6xaDYFYl1mVX4MaceaUUurNlbhDV7ixBqlTA52YFLJg/E6cPikZOd2WaTj+881dbWNnmemvE8NW9q7TylpKSgqqoKhYWFwjQdfZ58z9X8/Hz069dPiKbm56mioqLJc1WEptbOU0pKCkpKSoRqOvo8+Z6rFRUV6N27d9CbMjIy0F6d2ppr//79eO211/D222+joqICZ555Jj7//PN2f/zhw4fRt29fbNy4sckbL9x3331Yv349Nm/e3OJjNmzYgMsvvxw7d+5EbGwsrr32WlRUVGD58uWtfo3WXpn1nTTf3XEi/pQYrCbf17darf7Hm72p+dp9P336HidCU2uvFviOHb3Fipmbjj7uW6ssy7BarYZqSiusxmfbc/DFrnzkVfzvRjGHzYIzTojD2SPiMXVQL0SEhhzzuef7odL3tcx4npo3tfanPb41dWbtRmpqfp58f7VYLMK+Muv7k9Dmz1UzNzVfS1vM3NT8+PGeq1o3VVRUICYmpl27GXTo7Wx9hg4diqeeegqLFi3CF198gaVLl3bm0x1XVVUVrr76arzyyiuIjY1t18fY7XbY7fYWx30n42i+f4HNBXq8+eftyHFJkgI6rtXaj9fk9XqRmZmJ1NRUyLIsRFNr6/E1Hv0YMzc1P+71enHgwIEWjYB5m44+fvTzFDBW09CECDxw7nDcf84J+DXXiZW7DmPlrnwcdtbh81/z8fmv+bDIEsb1i8a01N6YNqQ3RvWNgiw33ddRVdVWn6dmOk/NNV+71+tFenp6q8/TjqzRCE3Njzd/rorQ1JrWnqtmbmq+Fq2fq0Zoan68s89VLZra0qlh9ugvOHv27IB3FYiNjYXFYmnyx0cAUFhYiISEhBaPz8jIQHZ2Ns477zz/Md9Eb7VasX//fgwaNCjwACKiLiZJEsYmR2NscjQePOcE7MytwMpd+fh+XxEyS1z4Jbscv2SX45k1aYhx2DBxQE+M69cTJ/bridFJUQjhlrZERAA0GmY7ymazYfz48Vi7dq1/EFYUBWvXrsXtt9/e4vHDhg3D7t27mxx7+OGHUVVVheeeew7JycldsWwiIk3JsoRx/RqH1Uf+OBw5ZTVYn1aMH9KKsTGjFGUuN1b/VojVvzX+4G+VJZyQGIEBEcCkMjsGx0diUJwDvcPtTS4bISLqDnQdZgFg/vz5uOaaazBhwgRMmjQJzz77LFwul393gzlz5qBv375YtGgRQkNDMXLkyCYfHx0dDQAtjhMRmVVyTBiuOqk/rjqpPxq8Cn7NqcC2g+XYfqgc2w9VoLiqHrvzKrEbwBf7fvd/XGSoFYPiwtE/JgzRYTb0DLOhpyME0WE2xITZMLxPJGIcNv3CiIiCQPdh9rLLLkNxcTEeffRRFBQUYOzYsfj6668RHx8PADh06FCb119Q15Nl2X+9rKjYaH4i9YVYZEwYEIMJA2IANF4rm1dRix2HyvFrTgUyi13IKHEhp6wGlXUe7DhUgR2HKtr4XBL+MCwOl4xPxvShvRFiMfa/H5HOY1vYKAY26qtTuxmYUSDv9UstqaoKt9sNm80m7B9nstH8RO8DWjbWNXiRXepCRpELeRU1KK9pQEWNG+WuBpTXuFFUVY+skv9toxMbbsPssX1x8YQkDEsw5vfC7ngeRcRGMXR1YyDzGodZCsjx7tgUARvNT/Q+oGON+woq8fHWXCzfmYeS6v/tA9kvJgynpsbi1NTemDq4FyJDjfEOZTyPYmCjGLq6MZB5TffLDIiIqGsMS4jEw38cjvvPGYb1+4vx0bYcfLevCIfKavDO5kN4Z/MhWGQJJyZH46SBvZAS60C/XmHoHxOG3hG8uYyIjInDLBFRNxNikTFjeDxmDI+Hq96DnzNL8WN6CX5IK0ZmiQtbD5Zj68HyJh8TGiIjuWcY4iLtCLdb4bBbEWG3IjzUisjQEExMicHYpOgW++ESEQUbh1kKmBEv/tYaG81P9D5Am0aH3YozTojHGSc03nSbU1aDH9NLsDuvAofKanCwtAaHK2pR16Agvaga6UXVbX6uhMhQnD0yATNHJGBSSgwsGgy2PI9iYKMYjNrIa2aJiOiYGrwK8sprcaisBuU1blTVeVBd70H1kb8WVtbhx/QSVNd7/B/Ty2HD6cPiMDwxEkPiI5AaH444XqpARO3Ea2YpaFRVhcvlgsPhEPZ/Smw0P9H7gK5tDLHIGBDrwIBYR5uPqWvw4qcDJfhqTwHW/F6IUpcbH2/LbfKYiFArUuPCMaCXA70j7P5fcRGhiIu0I7lnGGzW/73yw/MoBjaKwciNHGYpIIqiIDc3V+g7NtlofqL3AcZrDA2x+C9XaPAq+DmzFD9nluJAUTXSC6uRXepCVZ0H2w9VYHsb++BaZQmD48IxLCECwxIjMSTOgZCaEpw0ehhCQsT835XRzmMwsFEMRm4U87sDERHpJsQi49TU3jg1tbf/WL3Hi6wSF9IKq3G4ohZFlfUorq5HUWUdiqvqUVBZhxq3F/sKqrCvoArYedj/sdZPDiEhKhR9onqgT3Qo+kT3QHJMGFLjwpEaF4GoMGNsJUZE+uAwS0REQWe3WjAsIbLNN2hQVRWHnXXYl1+JfQVV2JtfiX35lcgsccGjqMgtr0VueW2rHxsXYUdqfONgO6pvFCYOiEFyTA/D/VEoEQUHh1kKiCRJQr/DCcBGEYjeB4jXKEkS+kb3QN/oHv6dFRRFQUZmFnrEJKCgsh6HnXU4XFGLwxW1yC6tQXphFfKddSiqqkdRVT1+OlDq/3xxEXZMHBCDCQN6Ynz/nhgcF44wm/H+lyfaeWwNG8Vg5EbuZkBERKZVVdfQeF1uUTX2F1Rh+6Fy7MlzosHb8n9tcRH2xhvZeoUd+asD/WLC0K9XmGHe9YyIGvHtbI+Bw2znqKoKp9OJqKgoQ/50pgU2mp/ofQAbj6WuwYtfcyqw9WA5fskuw685FSivaTjmx/QMC0G/Xg70jwlDckwP9Ilu/NX3yF/D7cF5VZfnUQxs1B635qKgURQFBQUFiIiIMNzdjFpho/mJ3gew8VhCQyyYPLAXJg/s5T/mrGlAdqmr8VdJjf/vD5XWoNTlRnlNA8prKvBrTkWrnzMy1Io+0T2Q1PN/g26f6B7oExWK2HA7eoXbEG63Bvw/eZ5HMbBRXxxmiYhIeFFhIRgTFo0xydEtfq+qrgGHympwqLQGB8tqkFfeeF1u3pHrcyvrPI2/fDsttMFulf2Dbb+YMIzsG4WRfaIwok8kejpsQawj6t44zBIRUbcWERqCEX2iMKJPVKu/X13vaTLc+obdwxV1yK+sRWm1GzVuL+o9CvKOPG5XrhNf7sr3f46+0T0wok8kEqNCER1mQ8+wEPR02BAZakF1aT2i4usQFxWmyVsAE3U3HGYpIJIkGfLdP7TERvMTvQ9gY1cKt1sxJD4CQ+Ij2nxMjduD0mo3Sl1uFFfV40BRNfYcduK3PCeyS2v8Q26bvsiFLAG9wu2Ij2x8V7Te4XbEhNvQy2FDr3AbYhx29HLYEBdpR6zDDtkkg69RzmMwsVFfvAGMiIgoiCrrGvD74Ursza9ESXU9ymsaUFHjRrmrAeU1jQNwaXU9lAD+bxxikRAXEYrEqFDER4UiITIU0T1CEBUWgqgejb+iw2zoEWKBJAGNc68ESQIkAD1sFoTbrXDYrKYZiql74W4Gx8BhtnMURUFZWRliYmIgy/LxP8CE2Gh+ovcBbBSFrzEquifKaxpQVFWPwsrGvXNLqupR6nKjzOVGqave/8pvSXU9tPw/d7jdinC7FWF2C1obayVJQohFhs3S+Ffrkb/2DLMhMapxoE448u5s8ZGhCLNZYLdaEGKRIElStzqPbNQOdzOgoFFVFSUlJejZs6feSwkaNpqf6H0AG0VxdGNcZCjiIkMxsm/r1+76NHgVFFfVI99ZhwJnHfKdtSiqqoezpgHO2sZfFbUNqKxtQG2DF6qqQgWgqo1fT1GB2gYvvEdeCq6u96C63hOUPrtVhs0qwy6riI9yoFeEHbHhNvQ+cqNcfGTj2xMnRjUOwiEWcw6C3e25ajQcZomIiEwkxCL7twbrKFVVUe9RUFXngevIMOtqY6D1qio8XhUeRYHbo6LBq8DtUVDmcuOwsxYFzjocdtah4MhQffSrxvUepfHrACipqQTyW/0SABovhYiPDEVCVChsR4Za3+WZEiTIcmN744Bsgc3SOChH9rAiIbLxUgvfJRdxEXZYTToYU+A4zBIREXUzkiQhNMSC0BALekfYNfu8iqLC7VWODLFeuD0KausbsPdAFsJiElBW04DS6sZLJUqq64+8stz46nKDVz3y93WdXockAQ6bFT1sFoTZLOgR0vhXh92K6DAbYo7sJhHjsKFnmA2RPUKaPC7syMeG263cYcIEOMxSQCRJEvodTgA2ikD0PoCNohCtUZYlhMqNQzLQ+BbBiqIgXI1HfHxcm9daKoqKElc9Dlc0XjrhVVQ0XhwB/yu9ypFXk92+X0deIa6oaUBBZeMrxIWVjdccexRVs8snIuxWRPYIafwVakVUjxDERtjRO9yO3hFHfoXboCAUVXUeRPaQhbypzsjPVd4ARkRERMJQFBWlLjeq6z2ocXtQ6/ai5sgvV70H5TVuVNQ0oKzGjXKXG+U1blTWelDb4EWN24Matxe1bi88gWwvcRRZaty72LerhM3acoCXAPR02BB3ZBiOiwhF74j/vZNc2JFXhcNs1lY/vjvgDWAUNIqioLCwEPHx8ULfsclGcxO9D2CjKNioPVmW/K+Ydobbo6CqrgGVdR7/jXWVR26uK6mqR3F1PYqr6lFUVY/iqjqUVtej3tN4g53v8VrwXRssSYBFlmCRJMiyBKssNV4WYW+8LMJhsyDMbkW4zYpoRwhiwmyNl1KE2dDTEYJeDjtiI+xw2CwdenXVyM9VDrMUEFVV4XQ6ERcXp/dSgoaN5id6H8BGUbDRuGxWGb3C7egVfvyh2Ov1Ij09Hf1SBqHa7UWlb1eJmgY0eFu+wquoqv8NNoqr6o78tXErthq3F9X1Hrg9CgA0Xk7hVTTr6hFiQWxE444SseF2xB15kw7/m3VE2BEXYUd4qPXIPsWNg6+RzyOHWSIiIiIN2K0ywuwhiIsI7fTnavAq/ksj3B4FiqpCUVV4FcCrqPAqKuo8jb/ve1yN24uqugaU1zSg3OX2X0rR+MYcbtQ2eFHb4EVOWS1yyo7xjnRH+G6kCztyI50VHrxybV+k9G773fD0wGGWiIiIyGBCLDKiesiI6hGi2ed01Xv8O0n4Xg0urqpHYWU9iqrqjvy1HqWu+iP7Erfch9iI97ZxmKWASJKE2NhYQ97NqBU2mp/ofQAbRcFGMZil0WG3wmG3on8vxzEfpxx51be63oOa+sa/Vtc1oLC0QpNXnbXG3QyIiIiIyFACmdeMdTsaGZ6iKMjJyYGiaHcxutGw0fxE7wPYKAo2ioGN+uIwSwFRVRUulwsiv6DPRvMTvQ9goyjYKAY26ovDLBERERGZFodZIiIiIjItDrMUEFmWkZCQYLh3/9ASG81P9D6AjaJgoxjYqC/uZkBEREREhsLdDChoFEVBZmamIe9m1AobzU/0PoCNomCjGNioLw6zFBBVVeF2uw15N6NW2Gh+ovcBbBQFG8XARn1xmCUiIiIi0+IwS0RERESmxWGWAiLLMpKSkgx5N6NW2Gh+ovcBbBQFG8XARn1xNwMiIiIiMhTuZkBB4/V6kZaWBq/Xq/dSgoaN5id6H8BGUbBRDGzUF4dZCpgRt+XQGhvNT/Q+gI2iYKMY2KgfDrNEREREZFocZomIiIjItHgDGAXEt2myzWaDJEl6Lyco2Gh+ovcBbBQFG8XARu3xBjAKKqvVqvcSgo6N5id6H8BGUbBRDGzUD4dZCoiiKEhPTzfsReBaYKP5id4HsFEUbBQDG/XFYZaIiIiITIvDLBERERGZFodZIiIiIjIt7mZAAVFVFYqiQJZloe/YZKO5id4HsFEUbBQDG7XH3QwoqDwej95LCDo2mp/ofQAbRcFGMbBRPxxmKSCKoiArK8uQdzNqhY3mJ3ofwEZRsFEMbNQXh1kiIiIiMi0Os0RERERkWhxmKWCyLP7Tho3mJ3ofwEZRsFEMbNQPdzMgIiIiIkPhbgYUNKqqorq6GiL/DMRG8xO9D2CjKNgoBjbqi8MsBURRFOTm5hrybkatsNH8RO8D2CgKNoqBjfriMEtEREREpsVhloiIiIhMi8MsBUSSJNhsNmHfrg9gowhE7wPYKAo2ioGN+uJuBkRERERkKNzNgIJGVVVUVFQY8m5GrbDR/ETvA9goCjaKgY364jBLAVEUBQUFBYa8m1ErbDQ/0fsANoqCjWJgo744zBIRERGRaXGYJSIiIiLT4jBLAZEkCQ6Hw5B3M2qFjeYneh/ARlGwUQxs1Bd3MyAiIiIiQ+FuBhQ0iqKgpKTEkBeAa4WN5id6H8BGUbBRDGzUF4dZCoiqqigpKTHk1hxaYaP5id4HsFEUbBQDG/XFYZaIiIiITIvDLBERERGZFodZCogkSYiKijLk3YxaYaP5id4HsFEUbBQDG/XF3QyIiIiIyFC4mwEFjaIoyM/PN+TdjFpho/mJ3gewURRsFAMb9cVhlgKiqiqcTqch72bUChvNT/Q+gI2iYKMY2KgvDrNEREREZFocZomIiIjItDjMUkAkSUJsbKwh72bUChvNT/Q+gI2iYKMY2Kgv7mZARERERIbC3QwoaBRFQU5OjiHvZtQKG81P9D6AjaJgoxjYqC8OsxQQVVXhcrkMeTejVthofqL3AWwUBRvFwEZ9cZglIiIiItPiMEtEREREpsVhlgIiyzISEhIgy+I+ddhofqL3AWwUBRvFwEZ9cTcDIiIiIjIU7mZAQaMoCjIzMw15N6NW2Gh+ovcBbBQFG8XARn1xmKWAqKoKt9ttyLsZtcJG8xO9D2CjKNgoBjbqi8MsEREREZkWh1kiIiIiMi0OsxQQWZaRlJRkyLsZtcJG8xO9D2CjKNgoBjbqi7sZEBEREZGhcDcDChqv14u0tDR4vV69lxI0bDQ/0fsANoqCjWJgo744zFLAjLgth9bYaH6i9wFsFAUbxcBG/XCYJSIiIiLT4jBLRERERKbFG8AoIL5Nk202GyRJ0ns5QcFG8xO9D2CjKNgoBjZqjzeAUVBZrVa9lxB0bDQ/0fsANoqCjWJgo344zFJAFEVBenq6YS8C1wIbzU/0PoCNomCjGNioLw6zRERERGRaHGaJiIiIyLQ4zBIRERGRaXE3AwqIqqpQFAWyLAt9xyYbzU30PoCNomCjGNioPe5mQEHl8Xj0XkLQsdH8RO8D2CgKNoqBjfrhMEsBURQFWVlZhrybUStsND/R+wA2ioKNYmCjvjjMEhEREZFpcZglIiIiItPiMEsBk2XxnzZsND/R+wA2ioKNYmCjfribAREREREZCnczoKBRVRXV1dUQ+WcgNpqf6H0AG0XBRjGwUV+GGGaXLFmCAQMGIDQ0FJMnT8aWLVvafOwrr7yCU089FT179kTPnj0xY8aMYz6etKUoCnJzcw15N6NW2Gh+ovcBbBQFG8XARn3pPsx+8MEHmD9/PhYsWIDt27djzJgxmDlzJoqKilp9/Lp163DFFVfg+++/x6ZNm5CcnIyzzjoLeXl5XbxyIiIiItKbVe8FLF68GDfeeCPmzp0LAHj55ZexcuVKLF26FA888ECLx7/zzjtN/vnVV1/FJ598grVr12LOnDktHl9fX4/6+nr/P1dWVgIAvF4vvF4vAECSJMiyDEVRmrx83tZx37tftHXc93mPPg6gxU8zbR23WCz+d9povpa2jrd37Z1t8nq9/sc0X4tZm5qv3dfo+30Rmpqv3Xf+mj/ezE1HHz/6HIrSdLSjm5p/HxOhycf3982fq2Zuan6efM9VRVFgsViEaGptja09V83edPRafI9p/rnN3NT8+PGeq1o3NX/8seg6zLrdbmzbtg0PPvig/5gsy5gxYwY2bdrUrs9RU1ODhoYGxMTEtPr7ixYtwsKFC1scz8jIQHh4OAAgKioKiYmJKCwshNPp9D8mNjYWsbGxyMvLg8vl8h9PSEhAdHQ0srOz4Xa7/ceTkpIQHh6OjIyMJicnJSUFVqsV6enpTdaQmpoKj8eDrKysJv1DhgyBy+VCbm6u/7jNZsPAgQPhdDpRUFDgP+5wOJCcnIyysjKUlJT4jwerSVVVOJ1ONDQ0QJIkIZqan6eMjAw4nU5kZGRAkiQhmpqfp0GDBsFisfgbRWg6+jz5nqdZWVkYOnSoEE0+vvNUW1vb5HkqQlPz8xQSEgKbzYbKysomf1pn5qbm58n3XD18+DD69+8vRFPz81ReXt7kuSpCU/PzFBkZCZvNhuLiYv+LZmZvan6efM/V8vJyxMXFBb0pIyMD7aXrbgaHDx9G3759sXHjRkyZMsV//L777sP69euxefPm436OW2+9FatXr8Zvv/2G0NDQFr/f2iuzvpPmuztOxJ8S2cQmNrGJTWxiE5vM2lRRUYGYmJh27Wag+2UGnfHkk0/i/fffx7p161odZAHAbrfDbre3OG6xWGCxWJoc8/0LbC7Q480/b0eOS5IU0HGt1n68Jt9PZlFRUW2u5ejHt2fteje19vijG4/1eLM0NT/e/DwezaxNRx8/ug8Qo6k1VVVVLc6hmZuar11VVVRUVCAqKkqYpubHmz9XRWhq7bhvm6Wjn6tmbmq+lqOfq62t04xNzY939rmq1fe9Vr9mux8ZBLGxsbBYLCgsLGxyvLCwEAkJCcf82KeffhpPPvkkvvnmG4wePTqYy6SjKIqCgoKCFj9JiYSN5id6H8BGUbBRDGzUl67DrM1mw/jx47F27Vr/MUVRsHbt2iaXHTT31FNP4e9//zu+/vprTJgwoSuWSkREREQGpPtlBvPnz8c111yDCRMmYNKkSXj22Wfhcrn8uxvMmTMHffv2xaJFiwAA//znP/Hoo4/i3XffxYABA/wXMYeHh/tv6CIiIiKi7kH3Yfayyy5DcXExHn30URQUFGDs2LH4+uuvER8fDwA4dOhQk+svXnrpJbjdblx88cVNPs+CBQvwt7/9rSuX3i1JkgSHw9HiOkuRsNH8RO8D2CgKNoqBjfrSdTcDPQTyXr9ERERE1PUCmdd0fwcwMhdFUVBSUmLIC8C1wkbzE70PYKMo2CgGNuqLwywFRFVVlJSUQOQX9NlofqL3AWwUBRvFwEZ9cZglIiIiItPiMEtEREREpsVhlgLie19tI97NqBU2mp/ofQAbRcFGMbBRX9zNgIiIiIgMhbsZUNAoioL8/HxD3s2oFTaan+h9ABtFwUYxsFFfHGYpIKqqwul0GvJuRq2w0fxE7wPYKAo2ioGN+uIwS0RERESmxWGWiIiIiEyLwywFRJIkxMbGGvJuRq2w0fxE7wPYKAo2ioGN+uJuBkRERERkKNzNgIJGURTk5OQY8m5GrbDR/ETvA9goCjaKgY364jBLAVFVFS6Xy5B3M2qFjeYneh/ARlGwUQxs1BeHWSIiIiIyLQ6zRERERGRaHGYpILIsIyEhAbIs7lOHjeYneh/ARlGwUQxs1Bd3MyAiIiIiQ+FuBhQ0iqIgMzPTkHczaoWN5id6H8BGUbBRDGzUF4dZCoiqqnC73Ya8m1ErbDQ/0fsANoqCjWJgo744zBIRERGRaXGYJSIiIiLT4jBLAZFlGUlJSYa8m1ErbDQ/0fsANoqCjWJgo764mwERERERGQp3M6Cg8Xq9SEtLg9fr1XspQcNG8xO9D2CjKNgoBjbqi8MsBcyI23JojY3mJ3ofwEZRsFEMbNQPh1kiIiIiMi0Os0RERERkWrwBjALi2zTZZrNBkiS9lxMUbDQ/0fsANoqCjWJgo/Z4AxgFldVq1XsJQcdG8xO9D2CjKNgoBjbqh8MsBURRFKSnpxv2InAtsNH8RO8D2CgKNoqBjfriMEtEREREpsVhloiIiIhMi8MsEREREZkWdzOggKiqCkVRIMuy0HdsstHcRO8D2CgKNoqBjdrjbgYUVB6PR+8lBB0bzU/0PoCNomCjGNioHw6zFBBFUZCVlWXIuxm1wkbzE70PYKMo2CgGNuqLwywRERERmRaHWSIiIiIyLQ6zFDBZFv9pw0bzE70PYKMo2CgGNuqHuxkQERERkaFwNwMKGlVVUV1dDZF/BmKj+YneB7BRFGwUAxv1xWGWAqIoCnJzcw15N6NW2Gh+ovcBbBQFG8XARn1xmCUiIiIi0+IwS0RERESmxWGWAiJJEmw2m7Bv1wewUQSi9wFsFAUbxcBGfXE3AyIiIiIyFO5mQEGjqioqKioMeTejVthofqL3AWwUBRvFwEZ9cZilgCiKgoKCAkPezagVNpqf6H0AG0XBRjGwUV8cZomIiIjItDjMEhEREZFpcZilgEiSBIfDYci7GbXCRvMTvQ9goyjYKAY26ou7GRARERGRoXA3AwoaRVFQUlJiyAvAtcJG8xO9D2CjKNgoBjbqi8MsBURVVZSUlBhyaw6tsNH8RO8D2CgKNoqBjfriMEtEREREpsVhloiIiIhMi8MsBUSSJERFRRnybkatsNH8RO8D2CgKNoqBjfribgZEREREZCjczYCCRlEU5OfnG/JuRq2w0fxE7wPYKAo2ioGN+uIwSwFRVRVOp9OQdzNqhY3mJ3ofwEZRsFEMbNQXh1kiIiIiMi0Os0RERERkWhxmKSCSJCE2NtaQdzNqhY3mJ3ofwEZRsFEMbNQXdzMgIiIiIkPhbgYUNIqiICcnx5B3M2qFjeYneh/ARlGwUQxs1BeHWQqIqqpwuVyGvJtRK2w0P9H7ADaKgo1iYKO+OMwSERERkWlxmCUiIiIi0+IwSwGRZRkJCQmQZXGfOmw0P9H7ADaKgo1iYKO+uJsBERERERkKdzOgoFEUBZmZmYa8m1ErbDQ/0fsANoqCjWJgo744zFJAVFWF2+025N2MWmGj+YneB7BRFGwUAxv1xWGWiIiIiEyLwywRERERmRaHWQqILMtISkoy5N2MWmGj+YneB7BRFGwUAxv1xd0MiIiIiMhQuJsBBY3X60VaWhq8Xq/eSwkaNpqf6H0AG0XBRjGwUV8cZilgRtyWQ2tsND/R+wA2ioKNYmCjfjjMEhEREZFpcZglIiIiItPiDWAUEN+myTabDZIk6b2coGCj+YneB7BRFGwUAxu1xxvAKKisVqveSwg6Npqf6H0AG0XBRjGwUT8cZikgiqIgPT3dsBeBa4GN5id6H8BGUbBRDGzUF4dZIiIiIjItDrNEREREZFocZomIiIjItLibAQVEVVUoigJZloW+Y5ON5iZ6H8BGUbBRDGzUHnczoKDyeDx6LyHo2Gh+ovcBbBQFG8XARv1wmKWAKIqCrKwsQ97NqBU2mp/ofQAbRcFGMbBRXxxmiYiIiMi0OMwSERERkWlxmKWAybL4Txs2mp/ofQAbRcFGMbBRP9zNgIiIiIgMhbsZUNCoqorq6mqI/DMQG81P9D6AjaJgoxjYqC8OsxQQRVGQm5tryLsZtcJG8xO9D2CjKNgoBjbqi8MsEREREZkWh1kiIiIiMi0OsxQQSZJgs9mEfbs+gI0iEL0PYKMo2CgGNuqLuxkQERERkaFwNwMKGlVVUVFRYci7GbXCRvMTvQ9goyjYKAY26ovDLAVEURQUFBQY8m5GrbDR/ETvA9goCjaKgY364jBLRERERKbFYZaIiIiITIvDLAVEkiQ4HA5D3s2oFTaan+h9ABtFwUYxsFFf3M2AiIiIiAyFuxlQ0CiKgpKSEkNeAK4VNpqf6H0AG0XBRjGwUV8cZikgqqqipKTEkFtzaIWN5id6H8BGUbBRDGzUF4dZIiIiIjItDrNEREREZFocZikgkiQhKirKkHczaoWN5id6H8BGUbBRDGzUF3czICIiIiJDMd1uBkuWLMGAAQMQGhqKyZMnY8uWLcd8/EcffYRhw4YhNDQUo0aNwqpVq7popaQoCvLz8w15N6NW2Gh+ovcBbBQFG8XARn3pPsx+8MEHmD9/PhYsWIDt27djzJgxmDlzJoqKilp9/MaNG3HFFVfg+uuvx44dOzB79mzMnj0be/bs6eKVd0+qqsLpdBrybkatsNH8RO8D2CgKNoqBjfrSfZhdvHgxbrzxRsydOxfDhw/Hyy+/jLCwMCxdurTVxz/33HM4++yzce+99+KEE07A3//+d4wbNw4vvvhiF6+ciIiIiPRm1fOLu91ubNu2DQ8++KD/mCzLmDFjBjZt2tTqx2zatAnz589vcmzmzJlYvnx5q4+vr69HfX29/5+dTicAoLy8HF6vF0DjRc2yLENRlCY/cbR1XJZlSJLU5nHf5z36OIAWL823ddxisUBV1SbHfWtp63h7197ZJq/Xi8rKSjidTlgsFiGamq+9oaEBlZWVKC8vh8ViEaKp+dpVVUVVVZW/UYSmo4/7nqfl5eWw2WxCNB3NYrHA4/E0eZ6K0NR87YqioLq6Gk6ns8lNJ2Zuan6efM/ViooKhISECNHU/Hjz76kiNDVfi++5WlFR4f86Zm9qfvx4z1WtmyoqKgCgXa8E6zrMlpSUwOv1Ij4+vsnx+Ph47Nu3r9WPKSgoaPXxBQUFrT5+0aJFWLhwYYvjAwYM6NiiiYiIiKhLVFVVISoq6piP0XWY7QoPPvhgk1dyFUVBWVkZevXqZcjtJYyusrISycnJyMnJEXY3CDaan+h9ABtFwUYxsFF7vj9B7NOnz3Efq+swGxsbC4vFgsLCwibHCwsLkZCQ0OrHJCQkBPR4u90Ou93e5Fh0dHTHF00AgMjISGH/g/Vho/mJ3gewURRsFAMbtXW8V2R9dL0BzGazYfz48Vi7dq3/mKIoWLt2LaZMmdLqx0yZMqXJ4wFgzZo1bT6eiIiIiMSl+2UG8+fPxzXXXIMJEyZg0qRJePbZZ+FyuTB37lwAwJw5c9C3b18sWrQIAHDXXXdh+vTpeOaZZzBr1iy8//772Lp1K/773//qmUFEREREOtB9mL3ssstQXFyMRx99FAUFBRg7diy+/vpr/01ehw4danJn4NSpU/Huu+/i4YcfxkMPPYTU1FQsX74cI0eO1CuhW7Hb7ViwYEGLSzdEwkbzE70PYKMo2CgGNuqr272dLRERERGJQ/c3TSAiIiIi6igOs0RERERkWhxmiYiIiMi0OMwSERERkWlxmCUiIiIi0+IwS0RERESmxWG2mysoKMC6desANL4PsoiKioqQnp6u9zKCio3mJ3ofwO83ougO57E7NGZnZ+Orr74C0Pjuq2bGYbYbUxQFL730ErZs2QIAkCRJ5xVpS1VVeL1e3HPPPfj+++/1Xk5QsNH8RO/z4fcbMYh+HoHu0QgA69evx7PPPgtFUUzfqPs7gJF+ZFnGySefjFdffdV/7IcffkCPHj0gyzLGjx+v4+o6T5IkWCwWnHPOOdixY4f/+Pbt2+FwOBAaGor+/fvruMLOY6P5G0Xv8+H3G55Hs+gOjQAwffp0fPvtt/53WV2xYgVycnIwatQoDB06FAkJCTqvsP04zHZzvXr1QmFhIQBg6dKlePrppzF48GCUl5fjggsuwD333KPzCjsvIiICP//8MwBgyZIleOWVV+BwONCnTx+cf/75uPrqq3VeYeex0fyNovcB/H7D82ge3aExKSkJ6enp2Lp1K37//Xc88cQTmDFjBj799FMMHz4cd999NwYPHqz3MttHpW4lKytLXbVqlfrll1/6j1166aXq1q1b1auuukpduXKlqqqqunz5cnXs2LHqBx98oNdSOywnJ0fduHGj+vPPP/uPXXrppWp2drZ6/vnnqxs2bFALCgrUJUuWqDNnzlTXrVun42o7ho3mbxS9T1X5/Ybn0Ty6Q+P+/fvVBQsWqP/973/VDRs2qKqqqjfddJP61VdfqcuWLVMff/xxVVVVdfXq1eqVV16pPvfcc6qqqqqiKLqtub14zWw3sm/fPowfPx6PP/44brrpJv8rBLIsY+fOnRg6dChWr16N8vJy/PGPf8To0aOxd+9enVcdmN9++w0TJkzA7bffjptuugkPPPAAAKCsrAw///wz+vXrh61btyI+Ph7/93//h4aGBvz+++86rzowbDR/o+h9AL/f8DyaR3dozMjIwJQpU7B161Z88skneOyxx3Dw4EFMmzYNy5cvR2FhIb755hsoioKzzjoLQ4cOxRdffAHAHNcMc5jtRl588UWcddZZ+OGHH7B69Wps3LgR27ZtwyWXXILS0lKMHz8ee/fuxVtvvYWMjAzIsozs7GzT3OWoKAoWLVqEc845B9988w2eeOIJrF27FhkZGZg9eza8Xi/Gjx+PZcuWYfny5QgJCUF8fDwyMjL0Xnq7sdH8jaL3+fD7Dc+jWXSHxo0bN2Ls2LH48ssv8eyzz8LhcOCnn35CSkoK8vPzMX/+fDidTsyaNQufffYZKisr4fV64XK59F56u/Ca2W4kKSkJhYWFyM/Px8iRIzF27Fi43W6EhYVh5cqVuO+++5CdnY2vv/4a//3vf+FyufDFF1/4Lw43OlmW0bt3b6iqisjISMyaNQsvvvgivF4voqKi8P777+Pzzz9Heno6nnrqKSxcuBD5+flYu3at3ktvNzaav1H0Ph9+v+F5NIvu0KgoCnJzc1FcXIxhw4YhKioKZWVluPLKK+HxeFBdXY1NmzbhiiuuwBtvvIGDBw/ijTfegMPh0Hvp7SKpqqAbqFELX3zxBZ577jn069cP4eHhWL16NX744QfExsbiyiuvxDvvvAOr1Yr8/HzU1tYiLCzMVHczAsCrr76K5557DhdffDFqa2vxxRdfYP369aivr8e8efPw4YcfAgB+//13uN1u9OrVC8nJyTqvOjBsNH+j6H0Av9/wPJpHd2gEGncvqKmpwZAhQ7B161Z888036N+/P2bOnIm5c+fi8ssvh6Io8Hq9qKurQ0REhN5Lbje+MtuNnHfeebBYLMjKykJBQQG+/fZbxMfHQ1VVFBYW4t1338WcOXMQHx9vqp84j3bDDTfAYrHg0KFDcLlcWLVqFWJjY9HQ0IDc3FysWbMGZ555Jk444QRTXAfUGjaav1H0PoDfb3gezUP0Rq/XC4vFgvXr1+PDDz+EzWbDv/71L/Tp0wcAcPHFF6OoqMj/+JCQEISEhOi13A7hMNtNqKoKSZJw7rnnNjleX18Pu92OWbNmobq6GgBM+R8r0PjHKLIsY+7cuQD+1+xrHD16NMrKygCY44L21rDR/I2i9wH8fsPzaB7dodFisfgH2ksvvdR/3HfM4XDggw8+wK233gqr1ZxjoTnPDLWb7yqStr6Z2u12AEBCQgJWrFiBhoYG0751X/NvNL5mX+Pw4cPx7bffwuv1stHARG8UuY/fb3gezaI7NAL/e5tai8XS4vd8xyZMmIBBgwahtra2S9emJV4zK5jS0lLk5eUhLCwM8fHxiIiI8L+CcCz5+fmQZRnx8fFdtNKOczqdKC8vR2RkJBwOB+x2u/8nzGPZunUr+vTp4/+jFSNjY9vM0ih6H8DvN8fC82gs3aExNzcXe/fuRVVVFSZMmIB+/foBwHGfr/X19XA6nYiLi+uqpWqOw6xAdu3ahcsuuwxutxsejweJiYl46aWXcOKJJ+q9NM3s3r0bc+bMQVVVFSwWi39vwJSUlHZ9YzIDNpq/UfQ+gN9veB7Nozs07t69G2eddRaSkpKwfft2TJgwAVOnTsW///1vAG0PtL7LLMzO/P8lEoDGnx5nzZqF888/H59//jmeeeYZ9OnTB1OnTsVnn32m9/I0kZOTgzPPPBOnn3463njjDdx0000oLCzEpEmTsGPHDsiyDK/Xq/cyO4WN5m8UvQ/g9xueR/PoDo1OpxNXXXUVLr/8cqxZswZZWVmYNWsWvvnmG1xwwQUA/nfdbHMiDLIA+Ha2ovjll1/UkSNHqgcPHvQfq66uVu+44w41NDRUXbNmjaqqqurxePRaYqetXr1anThxolpeXu4/tn//fvXCCy9UIyIi1N9++01VVVX1er06rbDz2Gj+RtH7VJXfb3gezaM7NGZlZalDhgxp8pbKlZWV6vvvv6+mpqaqV1xxhY6r6xocZgWxZs0aVZIkNTc3V1XV/32D9Xg86vXXX6/27NlTzc7O1nOJnfbee++pISEhallZWZPjhw4dUs8//3x12LBhan5+vk6r0wYbzd8oep+q8vsNz6N5dIfG0tJStX///urixYubHK+trVXfeOMNddSoUep///tfnVbXNXiZgSCmT5+Ok046CQ888ACcTidkWYaiKLBYLHj00Udxwgkn4J133gEAU96RCQDTpk3D6NGj8eyzz6KmpsZ/PCkpCffeey8cDgd++OEHHVfYeWw0f6PofQC/3/A8mkd3aAwLC8O0adOwZs0a/Pbbb/7joaGhuPjii9G/f3+sX79exxUGH4dZQVitVlx22WVIT0/HCy+8AJfL5b85oV+/fnA4HNi/fz8A814jk5CQgFNPPRWrVq3CZ599BrfbDaCx55RTTkF9fT1++uknnVfZOWw0f6PofQC/3/A8mkd3aAwNDcVf/vIXbNu2DY8//jgyMzP9v+dwODBt2jTs27fP1FtvHQ+HWQGoR+5GvO222zBx4kSsWLECTzzxRJMnblxcHHr16gVFUUz506eqqpBlGf/85z+RkJCAZ555Bm+99RYaGhr8jxk8eDASExN1XGXnsLGRmRtF7wP4/caH59H4ukMj0LiX7JgxY/DZZ59hxYoVeOihh5q8Epueno6kpKTjbidnZtyay8TUo7bU8G0T09DQgIcffhjff/89amtrccEFFyArKwuff/45Nm/ejOHDh+u86o7zbS1SX1+Pa665Bunp6UhISMDMmTOxe/dufPjhh9i8eTOGDRum91I7jI3mbxS1j99veB7NQvRGtZXttHydmzdvxk033YSQkBAoioL+/fvju+++w48//ojRo0frtOLg4zBrIvX19fB4PHA4HP5jR+8d53sye71erFu3Dh9++CGys7PRu3dv3H///Rg1apReS283r9cLRVGavC/00fs5+nobGhrw1ltv4auvvsKhQ4cQHx+Pxx9/3BT/sbLR/I2i9wH8fuP7fZ5H45/H7tDodDpRV1eHkJAQxMTE+I8fPdj6mrOzs7Ft2zZ89913SE5OxuzZs033Q1egOMyaxJ49e/Dwww8jKysLw4cPx6RJkzBv3jwATf9Dbf7HCGrjjhWm2Nz7999/x5NPPons7GyMGzcOU6dO9b+PtK+ttY3Ka2trYbFYYLPZ9Fh2QNho/kbR+wB+v+F5NM957A6Nu3fvxlVXXQVFUZCTk4O5c+fiwgsvxKmnngqgaUtrr9p2B8Y/i4QDBw5g2rRpSEhIwJVXXgmbzYbFixfj4osvBtD4HuEej8f/H+vu3bv9HytJkin+Y92/fz9OPvlkqKqKiRMnYvv27Xjsscdw5513Amjc8Nnj8fhbDhw44P/YHj16mOJ/LGw0f6PofQC/3wA8j2Y5j92hMTc3FzNmzMDpp5+O//f//h8WLlyIbdu24d5778Wnn34KoGnLl19+iZKSEj2XrI+gbPhFmnruuefUGTNmqA0NDaqqNm74/Pnnn6vx8fHqrFmz/I9TFEV9++23VYfDoX755Zd6LbdDHnnkEfXCCy/0/3NhYaH6/PPPq0lJSer111/f5LEvvfSSOmLECHXDhg1dvcxOYaP5G0XvU1V+v+F5NI/u0PjZZ5+pEyZMUGtra/3HNmzYoF599dXqqFGj1C+++MJ/fMWKFWpSUpL6yCOPmPrNPDrC+D+WEA4ePIiCggJYrVYAjVttnHvuuVi2bBm2bt2Km2++GUDjT2epqam44oorMGTIED2X/P/bu/eoqMr9f+DvAURRAREQ8IIk3s1RyBuiXyg00ExZZpJmRlqmHpE0iWOK92sdNfUo1skLneXdTto6ZXnXMC9BgjcCLyBeIA6KCqICM+/fH/xm5ygaKTrszee1Fkvn2ZuZz3v2Zs8ze579zF+Wnp5u9m6yXr16ePvttzF16lTs27cPM2bMUJa5u7ujWbNmaNCggSVKfWySUf0ZtZ4PkOONbEf1qAoZdTodzp07h4yMDKXN398fkZGRaNOmDWJjY3Hu3DkAQN++fREeHo7w8HBVnHWuUJbuTYs/t2vXLjZp0oTbtm0za79z5w5jY2Op1+v566+/mrWrhend4+rVq9mxY0f+8ssvZstzc3MZFRXF7t2788qVK0p7QUHBM63zSVSFjKYzI1rOSGo/H6nd443RaFT+L9tRvduR/OOYquWMJomJiWzRogVjY2OV46zJ9u3b6ebmxu+++85C1VUeVazrrh6857o8b29veHt7Y926dUhMTFTaq1evjuDgYJw/f16Z9NnUrhamd4/t2rXDjRs3sGrVKly5ckVZ7uzsjKFDhyI+Ph4nT55U2u+9arWyMhqNAP6YiFuv12su4/Xr1wFAOTPi6+urqYyZmZm4fPmycltr+QAgIyMDmzZtUuZQ1eLx5tSpU+jRowcKCgoAAD4+Pprbjrdv30ZxcbGyHRs3bqy57Whiet0wZVy7dq1mMt69exe3bt1S+gC+vr4IDg5GVFQU9uzZY7ZuSEgIvLy88N1331mi1MrF0r1pYS4rK0v5vu97x7zs3buX3t7eHDx4MH/66Sel/e7du/Tz8+OWLVueea2PKz09nf/61784depUfv/997x16xbJ0rFBNjY2jIiI4NmzZ5X1c3Nz6evrywMHDliq5L8sLS2NH374Id9++21OmjSJN2/eJElu3bpVMxlPnz7NatWqMSYmxqx927Ztmsh47Ngx6nQ6bt682axdK/lIMjk5mR4eHoyOjmZaWprSvnv3bs0cb5KSkuji4kKdTsd169Yp7VrajqdPn2afPn3YvXt3dujQQXkN0dJ2TEtL49SpUzlu3Dh+8sknvHr1Kkly3759bNKkCQcNGqT6jCdOnGC/fv2o1+v5xhtv8NNPP1WWDRw4kE5OTty6davySYHBYGDPnj05f/58S5VcaUhnthJJSUlh48aNGRoayosXL5IkS0pKlI/HduzYQb1ez6CgIM6bN4/x8fEcN24cnZ2dmZGRYcnSy+348eP08PBg79692bBhQ3bq1InTp09XPj7ZsmULHR0dGRYWxri4OKakpDAqKoru7u68dOmShasvn+PHj9PFxYVvvvkmX3nlFXbu3JmTJ09mSUkJSXLTpk2qz0iScXFxdHBwoKurK//+97+bLdu8eTMdHR05cOBAVWZMSkpi7dq1OWHChDKXb9iwQfXb8MKFC2zQoAHHjRtn1m463vz444/U6/V86aWXVHu8SUpKYo0aNRgdHc2QkBD26tXLbLkWjjenTp2is7MzR48ezU8//ZTdunVjYGAgi4qKSJZ+FK327XjixAk6OTlxyJAhfOWVV9ilSxfWr1+f+/btI1k63EDtGc+cOUMnJye+//77nDdvHocOHcqGDRsyNDRUWeett96ig4MDR4wYwZkzZzIiIoKOjo787bffLFh55SCd2Uri0qVL7Nq1K9u2bcuAgAAOGTKEmZmZJM07tEePHmVkZCQ9PDzYpk0b6vV6Hjt2zIKVl196ejqbNm3Kjz/+mMXFxbx9+zbHjRtHf39/s7FMu3btYmhoKF1dXdmyZUs2a9bMbNxTZXbmzBl6eXlx0qRJJMmioiIOGTKEEydOJPnH2NLt27erNqPJxo0b2aVLF65YsYJ169ZVMpocOHBAlRlPnjzJmjVrKnkMBgP379/Pr776ivv27WN+fj5J9W/DuLg4BgcHkyzNOHnyZL7zzjvs378/ExISSJaO11Pr8ebXX39ljRo1lDdae/fuZZ06dfif//yH5B+ddjUfbwoLC9m7d2+OHDlSafvyyy/57rvvsri4WDmuqnk75ufnMzAwkOPHjydZuq8eP36cjo6O9PT0VK7mP3TokGozkg+fmaFevXrs3bu3st5nn33GwYMHs3379uzXrx+TkpIsVXKlIp3ZSmLr1q0MCAjg4cOHGRsby27dupl1aIuLi5WDr8Fg4I0bN5iVlcXr169bsuxyKykp4YIFC9i/f3/+73//U85SnjlzhnXr1uXx48dJ/vECc/PmTWZmZjIlJYW5ubkWq/uvMBgMXLBgAd98803evHlTyTJy5Ei+9NJL7Nu3LwcOHKhcWHLt2jXVZbzXxYsX+dprrzE7O5uffPIJnZycOGfOHI4fP56xsbEk1bkdIyMjqdPpeOHCBZJkUFAQO3ToQDs7O7Zq1YrdunXjtWvXSJJ5eXmqy2fyySefKNNTde7cmUFBQRw6dCiDgoJYo0YNbty4kWTpG7KbN2+q6nhz9epVdujQgVFRUUpbZmYmO3bsyLFjx5I0P6aqcT8lS48hPj4+XL9+vdI2YcIEenp60sfHh97e3ty0aRPJ0rxqe90gS4fetW7dmrt37yZZepwtKSlhSEgI27Vrxzp16jA9PZ2kejOS5Pjx4/n888+btZWUlHDnzp10c3PjsGHDlHbTyaB7p+uq6qQzW4ns3LlT+f/y5cuVDq3pRdXUASSpujnkDAYDV61axc8//9ys/fLly6xTpw6PHDliocoq1qVLl5icnKzcnjdvHq2trRkTE8OYmBh269aNXl5evHHjhgWrrBgXL17kc889x5SUFN66dYtffvkl7ezsqNPplGEy9199qwaFhYUMDQ2lm5sbO3bsqJz9+P3337l161Z26tSJoaGhvHv3rqVLfSJLlixh48aN+d///pd9+/bljRs3lM7dmDFj6OTkxOzsbAtX+Xhu3779wEwFJLls2TLa2try9OnTJM1nOFCrwMBA6vV6HjhwgBMmTGCNGjW4YsUK7ty5kxEREaxVqxZTUlIsXeZju3btGvV6PadNm6a0ZWRk0MvLi3v27KGfnx/DwsJoMBhU97p4r/LMzJCYmEhSG/ttRZPObCXwsB2zrDO0M2fOVN07TpN730WaDjrFxcVs1aqV2cd627ZtUy4KU7OcnBz27NmTP/zwg9J25MgRuri48Mcff7RgZU/OtM/26dNH6Ri8/vrrdHR0pKOjo9kLjxrduXOHAwYMYKtWrXjmzBmzZQsWLGCzZs1UM6byYa5du8Zu3bqxefPmDAgIYFFRkTLOMj8/n56enmZn/NSirOOpqe3y5cvs1KkTJ06caDZ8S80OHTpEf39/hoaGsn79+lyxYoXZck9PT86aNctC1T25oqIijhs3jt27d+egQYO4YsUKOjg4cNSoUSTJuXPnslu3bqrclvfWnJ6ezp49ezIsLEwZ5mNy/vx51q5dW5V/j8+KTM1VCdz/PcqmKZ1GjhyJN998ExkZGZg0aRLCw8MxZcoUs6lk1KRGjRoAYPZ92MXFxbh9+zaKiooAAJMnT8bw4cM18XV8rq6u+OabbxAcHGw2TZebmxvq169v4eqejGmfdXV1RXx8PMLDwxEfH4+tW7dizpw5mD59OmbPnm3hKh9f9erVERcXh2XLlqFRo0YA/vi7bNiwIaysrFQxzc+j2Nvbo3///rh79y4yMzNBEtWqVQNQOj2Qs7Mz6tata+Eq/7qyvpfe1Fa/fn34+vpi8+bNMBqN0Ol0ZtMgqlGXLl0QHx+PlStXwtXVFXq9HkDpsTUvLw/u7u5o3Lixhat8PKZ9MiYmBsHBwbh8+TLWrl2L6OhoLF++HABQs2ZNs9cQNcjOzkZ2djZ0Op1yXPHy8sLHH3+MhIQELFy4EPHx8cr6DRo0QNu2bZW/T1EGy/alxcPc+3HJsmXLWLNmTdapU0dVA9rLIycnh46OjkxISODcuXNZvXr1B96Vqtn9ZwsmTpzIrl27qmpcXllM++f06dNpbW3Npk2bKmfXc3NzGRsbq9krbCMiIti7d29NfHpQUFDAmTNn0sHBgb6+vkxNTeXJkyc5Y8YMent7K8NFtMC0z2ZnZ7Nhw4aqPlv5MEFBQYyIiKDRaGRhYSFnzJhBLy8vZUypGpm2m+nfvLw8s+Xh4eEcPHiwaoY0VYVZiyxBOrOVmOmPNyIigg4ODjx58qSFK6p4hYWFfOGFF9ijRw/WqFGjzHFuWpCRkcGoqCg6OTmZjalVu+zsbA4YMOCBNyBqHrv2MOnp6YyKijK7YFHNTNuosLCQGzduZKdOnVizZk22bNmS3t7eqrmi/6+6c+cOX375Zfbr10+V3whVFqPRSIPBwHnz5tHHx4dubm7s0aMHPTw8NLsdf/nlF0ZFRdHBwYEnTpywdDnlUhVmLbIUHanyz1g0bseOHejfvz8OHDgAX19fS5dT4W7cuIGWLVvi1q1b+Omnn9CuXTtLl1Thjh49ivXr12PHjh1Yt26dZjIaDAZYW1sr/2rZ4cOHERcXh507d2LLli1o3769pUuqECTNPpY/cOAAnJ2d4ezsDHd3dwtW9nQlJiaidu3aaNGihaVLqVCFhYU4fPgw9uzZg4YNG+Lll19GkyZNLF1WhSsqKsLKlSuxdOlSrFu3TjV/j9u2bcOiRYswf/58HDt2DGvXroWXlxfmzJmDRo0aoaSkBNbW1srwg4KCAhQWFsLOzg6Ojo6WLr9Sk86sBRmNRmXs6KP8/vvvcHNzewYVVbw/y1hcXIy5c+di0KBBaNas2TOsrOL8WcY7d+4gISEBTZo0Ue1Y2fLuq2pVnm148OBBtGjRAg0bNnyGlVWcR2W8v1OrVuXZT9W+Lz+sfq1sQ6D82zEvLw/Ozs7PqKqKsWvXLvTo0QMAEBsbi3Xr1sHLywuzZ8+Gp6en2ckBte+rz5J0Zi3EtMPm5OQgLS0N/v7+ZV4IpuYd+c8ymg6+aj4IlzejmpVnX1Uz2YbaIBm1QasZH3YcWbFixQNnaGfNmoWIiAg5G/sXqLenpGJGoxHW1ta4cOECWrZsiYSEhDJ3cjV3ZMuT0XRbrQeqv5JRrcq7r6qVbENtkIzaoOWMVWXWIouxxEBdUXrhjL29PUeOHKnK+fHKQzJqg9Yzaj0fKRm1QjJqS1WZtehZkGEGFpKcnIydO3di/Pjxqj4D+yiSURu0nlHr+QDJqBWSUXtMwwnHjh2LuLg4/Pzzz2jTpo2ly1Id6cw+BUajESTNrvBW+/jX+0lGbdB6Rq3nAySjVkjGqkvrsxY9C9KZrWCnT5/GnDlzkJ2djWbNmqFPnz545ZVXAEAzUxhJRsmoBlrPB0hGyageVSFjWarCrEWVQdV+O1TBUlNT0bVrVxgMBnTs2BGHDh3CtGnTMG7cOABQ5uRUM8koGdVA6/kAyQhIRrWoChnLYjAYYGVlhZycHMTHx5f51cmmC8GkI/uEnv0wXW0yGo38+OOPOXDgQKXt5s2bnDVrFtu3b8/33nvPbF01koySUQ20no+UjJJRPapCxrKYLu7KyMigk5MTFy1aZNmCNE7OzFYQnU6HK1euIDs7W2mzt7fH2LFjMWTIEBw7dgzz589X1lUjySgZ1UDr+QDJKBnVoypkLIuVlRV+//13tG3bFmFhYYiMjLR0SZomndkKwP//0YGvry8MBgNSU1OVZfb29hg2bBh8fHzw7bffIj8/31JlPhHJKBnVQOv5AMkoGdWjKmR8lOzsbEyZMgXLli3TVEe9UrLYOWENOnv2LF1cXDhs2DDm5+eT/ONjk8zMTOp0Om7fvt2SJT4xySgZ1UDr+UjJKBnVQ4sZDQYDS0pKHmgTlmFj6c60lnh7e2PTpk3o1asX7OzsMG3aNLi4uAAAqlWrBr1er/qvp5OMklENtJ4PkIySUT20lvFhMzNYWVlpemaGykw6sxXsxRdfxObNm/H6668jKysLAwcOhF6vx1dffYWcnBw0atTI0iU+MckoGdVA6/kAySgZ1UMrGU0zM/Tq1QsdO3bE9u3bkZCQgF27dmHRokXKzAzSoX22ZJ7Zp+TXX3/F+PHjkZGRARsbG1hbW2PDhg3w8fGxdGkVRjJqg9Yzaj0fIBm1QjJWbiQxefJknD17Fhs3bgQA5OfnY8mSJdiyZQs6duyIL774QllXxsk+O9KZfYpu3ryJa9euIT8/Hx4eHsrHKloiGbVB6xm1ng+QjFohGSu3d955B+fPn8f+/fuVtvz8fHzxxRfYsGEDBgwYgOjoaAtWWDVJZ1YIIYQQ4hFMZ1qXLl2KjRs3YuXKlWjRooWyPC8vD9HR0Th16hR++OEH2NvbW7Daqkc6s0IIIYQQ5XDu3Dl06dIFffv2xeLFi1G7dm2lo3vx4kU0btwY33//PUJCQixdapUiF4AJIYQQQpSD1mZm0ArpzAohhBBClJNWZmbQEhlmIIQQQgjxF6l5Zgatkc6sEEIIIcRjUPPMDFoinVkhhBBCCKFaVpYuQAghhBBCiMclnVkhhBBCCKFa0pkVQgghhBCqJZ1ZIYQQQgihWtKZFUIIIYQQqiWdWSGEEEIIoVrSmRVCCCGEEKolnVkhhBBCCKFa0pkVQoinYN++fdDpdLh+/bqlS6lQXl5e+Oyzz5TbOp0OW7dufeZ1TJs2De3bt3/mjyuEqHykMyuE0JTw8HDodLoHfs6ePfvUHjMwMBAffPCBWVvXrl2RlZUFR0fHp/a4lUFWVhZ69epVrnWlAyqEeBpsLF2AEEJUtJCQEKxevdqszdXV9YH1ioqKYGtr+1RqsLW1hbu7+1O57ydVkbkra0YhRNUhZ2aFEJpTvXp1uLu7m/1YW1sjMDAQY8aMwQcffAAXFxcEBwcDABYuXIi2bduiVq1aaNSoEUaPHo2CggKz+zx48CACAwNRs2ZNODk5ITg4GHl5eQgPD8f+/fuxePFi5SxwRkZGmcMMvv76a7Rp0wbVq1eHl5cXFixYYPYYXl5emDNnDoYNGwZ7e3t4enriiy++eGRWU6YxY8bA0dERLi4uiImJAUmz+505cyaGDh0KBwcHjBgxAgAQHx+P7t27w87ODo0aNcLYsWNx69Yt5fdycnLw6quvws7ODs899xzWrl37wOPfP8zg0qVLGDRoEOrWrYtatWqhQ4cOOHLkCNasWYPp06cjOTlZeZ7WrFkDALh+/TreffdduLq6wsHBAS+99BKSk5PNHmfevHlwc3ODvb09hg8fjjt37jzyeRFCVB3SmRVCVClxcXGwtbXFwYMHsWLFCgCAlZUVlixZglOnTiEuLg579uzBRx99pPxOUlISgoKC0Lp1axw6dAjx8fF49dVXYTAYsHjxYvj5+eG9995DVlYWsrKy0KhRowceNzExEQMHDsQbb7yBEydOYNq0aYiJiVE6dCYLFixAhw4dcOzYMYwePRqjRo1Camrqn2aysbHB0aNHsXjxYixcuBBffvml2Tr/+Mc/0K5dOxw7dgwxMTE4d+4cQkJC8Nprr+H48ePYuHEj4uPjMWbMGOV3wsPDcfHiRezduxdbtmzB8uXLkZOT89A6CgoKEBAQgMuXL+Pbb79FcnIyPvroIxiNRoSFheHDDz9EmzZtlOcpLCwMAPD6668jJycH27dvR2JiInx9fREUFIRr164BADZt2oRp06Zhzpw5SEhIgIeHB5YvX/7I50QIUYVQCCE05O2336a1tTVr1aql/AwYMIAkGRAQQB8fnz+9j82bN9PZ2Vm5PWjQIPr7+z90/YCAAEZGRpq17d27lwCYl5dHkhw8eDB79uxptk5UVBRbt26t3G7cuDGHDBmi3DYajaxXrx5jY2Mf+ditWrWi0WhU2qKjo9mqVSuz+w0NDTX7veHDh3PEiBFmbT/99BOtrKx4+/ZtpqamEgCPHj2qLE9JSSEALlq0SGkDwG+++YYk+fnnn9Pe3p5Xr14ts9apU6eyXbt2Dzymg4MD79y5Y9bu7e3Nzz//nCTp5+fH0aNHmy3v3LnzA/clhKia5MysEEJzXnzxRSQlJSk/S5YsUZa98MILD6y/a9cuBAUFoUGDBrC3t8dbb72Fq1evorCwEMAfZ2afREpKCvz9/c3a/P39cebMGRgMBqVNr9cr/9fpdHB3d3/k2VAA6NKlC3Q6nXLbz8/vgfvt0KGD2e8kJydjzZo1qF27tvITHBwMo9GI9PR0pKSkwMbGxuz5atmyJerUqfPQOpKSkuDj44O6des+st776ygoKICzs7NZLenp6Th37hyA0ueuc+fOZr/n5+dX7scQQmibXAAmhNCcWrVqoWnTpg9ddq+MjAz06dMHo0aNwuzZs1G3bl3Ex8dj+PDhKCoqQs2aNWFnZ/csygYAVKtWzey2TqeD0Wh84vu9P3dBQQHef/99jB079oF1PT09kZaW9pcf43Gep4KCAnh4eGDfvn0PLHtUx1kIIUzkzKwQokpLTEyE0WjEggUL0KVLFzRv3hxXrlwxW0ev12P37t0PvQ9bW1uzs6BladWqFQ4ePGjWdvDgQTRv3hzW1taPHwDAkSNHzG4fPnwYzZo1e+T9+vr64vTp02jatOkDP7a2tmjZsiVKSkqQmJio/E5qauoj583V6/VISkpSxrrer6znydfXF9nZ2bCxsXmgDhcXFwClz11ZGYUQApDOrBCiimvatCmKi4uxdOlSnD9/Hv/+97+VC8NMJk6ciF9++QWjR4/G8ePH8dtvvyE2Nha5ubkASmcLOHLkCDIyMpCbm1vmmdQPP/wQu3fvxsyZM5GWloa4uDj885//xIQJE544Q2ZmJsaPH4/U1FSsX78eS5cuRWRk5CN/Jzo6Gj///DPGjBmDpKQknDlzBtu2bVMuAGvRogVCQkLw/vvv48iRI0hMTMS77777yLOvgwYNgru7O0JDQ3Hw4EGcP38eX3/9NQ4dOgSg9HlKT09HUlIScnNzcffuXfTo0QN+fn4IDQ3Fjh07kJGRgZ9//hmTJk1CQkICACAyMhKrVq3C6tWrkZaWhqlTp+LUqVNP/LwJIbRBOrNCiCqtXbt2WLhwIebPn4/nn38ea9euxdy5c83Wad68OXbs2IHk5GR06tQJfn5+2LZtG2xsSkdqTZgwAdbW1mjdujVcXV2RmZn5wOP4+vpi06ZN2LBhA55//nlMmTIFM2bMQHh4+BNnGDp0KG7fvo1OnTrhb3/7GyIjI5Xptx5Gr9dj//79SEtLQ/fu3eHj44MpU6agfv36yjqrV69G/fr1ERAQgP79+2PEiBGoV6/eQ+/T1tYWO3bsQL169dC7d2+0bdsW8+bNU84Qv/baawgJCcGLL74IV1dXrF+/HjqdDt9//z3+7//+D++88w6aN2+ON954AxcuXICbmxsAICwsDDExMfjoo4/wwgsv4MKFCxg1atQTP29CCG3QkfdMRiiEEEJVAgMD0b59e7OvmBVCiKpEzswKIYQQQgjVks6sEEIIIYRQLRlmIIQQQgghVEvOzAohhBBCCNWSzqwQQgghhFAt6cwKIYQQQgjVks6sEEIIIYRQLenMCiGEEEII1ZLOrBBCCCGEUC3pzAohhBBCCNWSzqwQQgghhFCt/we/dn4EO/hcdAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ms2query.benchmarking.AnnotatedSpectrumSet import AnnotatedSpectrumSet\n", + "\n", + "\n", + "plt.figure(figsize=(8, 8))\n", + "plot_comparison_plot({\n", + " # \"MS2DeepScore\": plot_average_per_threshold(ms2deepscores.squeeze(), neg_neg_tanimoto_for_prediction_ms2deepscore, neg_val_spectra, 100),\n", + " \"MS2Query 2.0\": plot_average_per_threshold(predicted_scores, neg_neg_tanimoto_for_prediction, AnnotatedSpectrumSet.create_spectrum_set(neg_val_spectra), 100),\n", + "})" ] } ], "metadata": { "kernelspec": { - "display_name": "ms2query_2", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, From 8c7af37f27c63e2aab4ed9b3c32ecc226457f3af Mon Sep 17 00:00:00 2001 From: Florian Huber Date: Tue, 31 Mar 2026 12:05:03 +0200 Subject: [PATCH 25/25] add highest predicted tanimoto score to results --- ms2query/ms2query_development/ReferenceLibrary.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ms2query/ms2query_development/ReferenceLibrary.py b/ms2query/ms2query_development/ReferenceLibrary.py index d9fa9b4..e12c91c 100644 --- a/ms2query/ms2query_development/ReferenceLibrary.py +++ b/ms2query/ms2query_development/ReferenceLibrary.py @@ -142,7 +142,9 @@ def run_ms2query( num_of_query_embeddings = query_embeddings.embeddings.shape[0] library_index_highest_ms2deepscore = np.zeros((num_of_query_embeddings), dtype=int) + highest_ms2deepscore_values = np.zeros((num_of_query_embeddings), dtype=float) ms2query_scores = [] + for start_idx in tqdm( range(0, num_of_query_embeddings, batch_size), desc="Predicting highest ms2deepscore per batch of " @@ -153,20 +155,29 @@ def run_ms2query( end_idx = min(start_idx + batch_size, num_of_query_embeddings) selected_query_embeddings = query_embeddings.embeddings[start_idx:end_idx] score_matrix = cosine_similarity_matrix(selected_query_embeddings, self.reference_embeddings.embeddings) + highest_score_idx = np.argmax(score_matrix, axis=1) + highest_score_values = np.max(score_matrix, axis=1) + library_index_highest_ms2deepscore[start_idx:end_idx] = highest_score_idx + highest_ms2deepscore_values[start_idx:end_idx] = highest_score_values # get predicted inchikeys predicted_inchikeys = self.reference_metadata.iloc[highest_score_idx]["inchikey"] + # Compute MS2Query reliability score ms2query_scores.extend( get_ms2query_reliability_prediction( - predicted_inchikeys, self.spectrum_indices_per_inchikey, self.top_k_tanimoto_scores, score_matrix + predicted_inchikeys, + self.spectrum_indices_per_inchikey, + self.top_k_tanimoto_scores, + score_matrix, ) ) # construct results df - results = self.reference_metadata.iloc[library_index_highest_ms2deepscore] + results = self.reference_metadata.iloc[library_index_highest_ms2deepscore].copy() + results["predicted_tanimoto"] = highest_ms2deepscore_values results["ms2query_reliability_prediction"] = ms2query_scores return results