From 961621b1b60e1f61d38b3beb7cd4adf0d92ec5ec Mon Sep 17 00:00:00 2001 From: Leo <81079993+LeoGitGuy@users.noreply.github.com> Date: Mon, 2 Jan 2023 13:19:12 +0300 Subject: [PATCH 1/2] Adjust max token size for openai ADA-v2 embeddings --- haystack/nodes/retriever/_embedding_encoder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/haystack/nodes/retriever/_embedding_encoder.py b/haystack/nodes/retriever/_embedding_encoder.py index 775c813cbea..4650ecebfb5 100644 --- a/haystack/nodes/retriever/_embedding_encoder.py +++ b/haystack/nodes/retriever/_embedding_encoder.py @@ -400,11 +400,11 @@ def __init__(self, retriever: "EmbeddingRetriever"): model_class: str = next( (m for m in ["ada", "babbage", "davinci", "curie"] if m in retriever.embedding_model), "babbage" ) - self._setup_encoding_models(model_class, retriever.embedding_model) + self._setup_encoding_models(model_class, retriever.embedding_model, retriever.max_seq_len) self.tokenizer = AutoTokenizer.from_pretrained("gpt2") - def _setup_encoding_models(self, model_class: str, model_name: str): + def _setup_encoding_models(self, model_class: str, model_name: str, max_seq_len: int): """ Setup the encoding models for the retriever. """ @@ -412,6 +412,7 @@ def _setup_encoding_models(self, model_class: str, model_name: str): if "text-embedding" in model_name: self.query_encoder_model = model_name self.doc_encoder_model = model_name + self.max_seq_len = min(8191, max_seq_len) else: self.query_encoder_model = f"text-search-{model_class}-query-001" self.doc_encoder_model = f"text-search-{model_class}-doc-001" From a802c1760b87c919343a3f5d5b6dbefc9e2d5901 Mon Sep 17 00:00:00 2001 From: Leo <81079993+LeoGitGuy@users.noreply.github.com> Date: Wed, 4 Jan 2023 16:54:38 +0300 Subject: [PATCH 2/2] Added requested changes and corrected old seq len Apparently the limit for the older models is 2046 and not 2048, I included this change directly. See (https://beta.openai.com/docs/guides/embeddings/what-are-embeddings) to check. --- haystack/nodes/retriever/_embedding_encoder.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/haystack/nodes/retriever/_embedding_encoder.py b/haystack/nodes/retriever/_embedding_encoder.py index 4650ecebfb5..d12812ebd54 100644 --- a/haystack/nodes/retriever/_embedding_encoder.py +++ b/haystack/nodes/retriever/_embedding_encoder.py @@ -391,8 +391,6 @@ def save(self, save_dir: Union[Path, str]): class _OpenAIEmbeddingEncoder(_BaseEmbeddingEncoder): def __init__(self, retriever: "EmbeddingRetriever"): # See https://beta.openai.com/docs/guides/embeddings for more details - # OpenAI has a max seq length of 2048 tokens and unknown max batch size - self.max_seq_len = min(2048, retriever.max_seq_len) self.url = "https://api.openai.com/v1/embeddings" self.api_key = retriever.api_key self.batch_size = min(64, retriever.batch_size) @@ -416,6 +414,7 @@ def _setup_encoding_models(self, model_class: str, model_name: str, max_seq_len: else: self.query_encoder_model = f"text-search-{model_class}-query-001" self.doc_encoder_model = f"text-search-{model_class}-doc-001" + self.max_seq_len = min(2046, max_seq_len) def _ensure_text_limit(self, text: str) -> str: """