forked from langchain-ai/langchainjs
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add initial voy implementation (langchain-ai#1181)
* add initial voy implementation * update implementation and tests * add docs and examples * fix lint * Rename to avoid conflicts with SDK, add delete * Mark Voy as optional * Lint --------- Co-authored-by: harry_squater <[email protected]> Co-authored-by: jacoblee93 <[email protected]>
- Loading branch information
3 people
authored
Sep 8, 2023
1 parent
570d052
commit 9f9a92a
Showing
15 changed files
with
421 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| diff --git a/package.json b/package.json | ||
| index 1823128f20579c0c70acfb394d84d5f2997aafb9..663f5b611870e762cf355b1d4810765f9c89b318 100644 | ||
| --- a/package.json | ||
| +++ b/package.json | ||
| @@ -1,5 +1,7 @@ | ||
| { | ||
| "name": "voy-search", | ||
| + "type": "module", | ||
| + "main": "voy_search.js", | ||
| "collaborators": [ | ||
| "Daw-Chih Liou <[email protected]>" | ||
| ], |
18 changes: 18 additions & 0 deletions
18
docs/extras/modules/data_connection/vectorstores/integrations/voy.mdx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| import CodeBlock from "@theme/CodeBlock"; | ||
|
|
||
| # Voy | ||
|
|
||
| [Voy](https://github.com/tantaraio/voy) is a WASM vector similarity search engine written in Rust. | ||
| It's supported in non-Node environments like browsers. You can use Voy as a vector store with LangChain.js. | ||
|
|
||
| ### Install Voy | ||
|
|
||
| ```bash npm2yarn | ||
| npm install voy-search | ||
| ``` | ||
|
|
||
| ## Usage | ||
|
|
||
| import Example from "@examples/indexes/vector_stores/voy.ts"; | ||
|
|
||
| <CodeBlock language="typescript">{Example}</CodeBlock> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| import { VoyVectorStore } from "langchain/vectorstores/voy"; | ||
| import { Voy as VoyClient } from "voy-search"; | ||
| import { OpenAIEmbeddings } from "langchain/embeddings/openai"; | ||
| import { Document } from "langchain/document"; | ||
|
|
||
| // Create Voy client using the library. | ||
| const voyClient = new VoyClient(); | ||
| // Create embeddings | ||
| const embeddings = new OpenAIEmbeddings(); | ||
| // Create the Voy store. | ||
| const store = new VoyVectorStore(voyClient, embeddings); | ||
|
|
||
| // Add two documents with some metadata. | ||
| await store.addDocuments([ | ||
| new Document({ | ||
| pageContent: "How has life been treating you?", | ||
| metadata: { | ||
| foo: "Mike", | ||
| }, | ||
| }), | ||
| new Document({ | ||
| pageContent: "And I took it personally...", | ||
| metadata: { | ||
| foo: "Testing", | ||
| }, | ||
| }), | ||
| ]); | ||
|
|
||
| const model = new OpenAIEmbeddings(); | ||
| const query = await model.embedQuery("And I took it personally"); | ||
|
|
||
| // Perform a similarity search. | ||
| const resultsWithScore = await store.similaritySearchVectorWithScore(query, 1); | ||
|
|
||
| // Print the results. | ||
| console.log(JSON.stringify(resultsWithScore, null, 2)); | ||
| /* | ||
| [ | ||
| [ | ||
| { | ||
| "pageContent": "And I took it personally...", | ||
| "metadata": { | ||
| "foo": "Testing" | ||
| } | ||
| }, | ||
| 0 | ||
| ] | ||
| ] | ||
| */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| import { expect, test } from "@jest/globals"; | ||
| import { Voy as VoyOriginClient } from "voy-search"; | ||
| import { Document } from "../../document.js"; | ||
| import { OpenAIEmbeddings } from "../../embeddings/openai.js"; | ||
| import { VoyVectorStore } from "../voy.js"; | ||
|
|
||
| const client = new VoyOriginClient(); | ||
|
|
||
| test("it can create index using Voy.from text, add new elements to the index and get queried documents", async () => { | ||
| const vectorStore = await VoyVectorStore.fromTexts( | ||
| ["initial first page", "initial second page"], | ||
| [{ id: 1 }, { id: 2 }], | ||
| new OpenAIEmbeddings(), | ||
| client | ||
| ); | ||
| // the number of dimensions is produced by OpenAI | ||
| expect(vectorStore.numDimensions).toBe(1536); | ||
| await vectorStore.addDocuments([ | ||
| new Document({ | ||
| pageContent: "added first page", | ||
| metadata: { id: 5 }, | ||
| }), | ||
| new Document({ | ||
| pageContent: "added second page", | ||
| metadata: { id: 4 }, | ||
| }), | ||
| new Document({ | ||
| pageContent: "added third page", | ||
| metadata: { id: 6 }, | ||
| }), | ||
| ]); | ||
| expect(vectorStore.docstore.length).toBe(5); | ||
| await vectorStore.addDocuments([ | ||
| new Document({ | ||
| pageContent: "added another first page", | ||
| metadata: { id: 7 }, | ||
| }), | ||
| ]); | ||
| const results = await vectorStore.similaritySearchWithScore("added first", 6); | ||
| expect(results.length).toBe(6); | ||
| await vectorStore.delete({ | ||
| deleteAll: true, | ||
| }); | ||
| const results2 = await vectorStore.similaritySearchWithScore( | ||
| "added first", | ||
| 6 | ||
| ); | ||
| expect(results2.length).toBe(0); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| import { test, expect } from "@jest/globals"; | ||
| import { Document } from "../../document.js"; | ||
| import { FakeEmbeddings } from "../../embeddings/fake.js"; | ||
| import { VoyVectorStore, VoyClient } from "../voy.js"; | ||
|
|
||
| const fakeClient: VoyClient = { | ||
| index: ({ embeddings }) => embeddings.map((i) => i.id).join(","), | ||
| add: (_) => {}, | ||
| search: () => ({ | ||
| neighbors: [ | ||
| { id: "0", title: "", url: "" }, | ||
| { id: "1", title: "", url: "" }, | ||
| ], | ||
| }), | ||
| clear: () => {}, | ||
| }; | ||
|
|
||
| test("it can create index using Voy.from text, add new elements to the index and get queried documents", async () => { | ||
| const vectorStore = await VoyVectorStore.fromTexts( | ||
| ["initial first page", "initial second page"], | ||
| [{ id: 1 }, { id: 2 }], | ||
| new FakeEmbeddings(), | ||
| fakeClient | ||
| ); | ||
|
|
||
| // the number of dimensions is produced by fake embeddings | ||
| expect(vectorStore.numDimensions).toBe(4); | ||
| await vectorStore.addVectors( | ||
| [ | ||
| [0, 1, 0, 0], | ||
| [1, 0, 0, 0], | ||
| [0.5, 0.5, 0.5, 0.5], | ||
| ], | ||
| [ | ||
| new Document({ | ||
| pageContent: "added first page", | ||
| metadata: { id: 5 }, | ||
| }), | ||
| new Document({ | ||
| pageContent: "added second page", | ||
| metadata: { id: 4 }, | ||
| }), | ||
| new Document({ | ||
| pageContent: "added third page", | ||
| metadata: { id: 6 }, | ||
| }), | ||
| ] | ||
| ); | ||
| expect(vectorStore.docstore.length).toBe(5); | ||
| const results = await vectorStore.similaritySearchVectorWithScore( | ||
| [1, 0, 0, 0], | ||
| 3 | ||
| ); | ||
| expect(results[0][0].metadata.id).toBe(1); | ||
| expect(results[1][0].metadata.id).toBe(2); | ||
| }); |
Oops, something went wrong.