Added updated docs, readme, chunkers and updates to cognify

topoteretes · Apr 8, 2024 · 2e641de · 2e641de
1 parent 3158e13
commit 2e641de
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 4 deletions.
diff --git a/cognee/api/v1/cognify/cognify.py b/cognee/api/v1/cognify/cognify.py
@@ -180,6 +180,8 @@ async def generate_graph_per_layer(text_input: str, layers: List[str], response_
             'document_id': file_metadata['id'],
             'layer_id': base_node_for_graph
         },]
+
+
     db_engine.load_cognify_data(data)
 
 
@@ -204,6 +206,8 @@ async def generate_graph_per_layer(text_input: str, layers: List[str], response_
     await add_propositions(nodes_by_layer_for_processing_doc)
 
     relevant_documents_to_connect = db_engine.fetch_cognify_data(excluded_document_id=file_metadata['id'])
+
+    print("Relevant documents to connect are: ", relevant_documents_to_connect)
     list_of_nodes =[]
     #
     # relevant_documents_to_connect=[  {'document_id': '6dfe01b6-07d2-5b77-83c8-1d6c11ce2aa7', 'layer_id': 'LLM_CLASSIFICATION_LAYER_Articles, essays, and reports_DOCUMENT_6dfe01b6-07d2-5b77-83c8-1d6c11ce2aa7', 'created_at': '2024-04-05 16:47:09.651000', 'updated_at': '2024-04-05 16:47:09.651000'}]

diff --git a/cognee/config.py b/cognee/config.py
@@ -106,7 +106,7 @@ class Config:
         db_name: str = os.getenv("POSTGRES_DB")
 
     # Model parameters and configuration for interlayer scoring
-    intra_layer_score_treshold: float = 0.9
+    intra_layer_score_treshold: float = 0.98
 
 
     # Client ID

diff --git a/cognee/infrastructure/databases/relational/duckdb/DuckDBAdapter.py b/cognee/infrastructure/databases/relational/duckdb/DuckDBAdapter.py
@@ -62,7 +62,7 @@ def fetch_cognify_data(self, excluded_document_id: str):
         self.db_client.execute(create_table_sql)
 
         # SQL command to select data from the 'cognify' table
-        select_data_sql = f"SELECT document_id, layer_id, created_at, updated_at, processed FROM cognify WHERE document_id != '{excluded_document_id}' AND processed = FALSE;"
+        select_data_sql = f"SELECT document_id, layer_id, created_at, updated_at, processed FROM cognify WHERE document_id != '{excluded_document_id}' AND processed = FALSE  ;"
 
         # Execute the query and fetch the results
         records = self.db_client.sql(select_data_sql).to_df().to_dict("records")
@@ -72,7 +72,7 @@ def fetch_cognify_data(self, excluded_document_id: str):
             # Fetching document_ids from the records to update the 'processed' column
             document_ids = tuple(record['document_id'] for record in records)
             # SQL command to update the 'processed' column to 'True' for fetched records
-            update_data_sql = f"UPDATE cognify SET processed = TRUE, document_id_target = {excluded_document_id} WHERE document_id IN {document_ids};"
+            update_data_sql = f"UPDATE cognify SET processed = TRUE WHERE document_id IN {document_ids};"
             # Execute the update query
             self.db_client.execute(update_data_sql)
 

diff --git a/cognee/modules/cognify/llm/resolve_cross_graph_references.py b/cognee/modules/cognify/llm/resolve_cross_graph_references.py
@@ -43,7 +43,7 @@ async def get_nodes_by_layer(layer_id: str, layer_nodes: List):
     score_points = await vector_engine.batch_search(
         layer_id,
         list(map(lambda node: node["description"], layer_nodes)),
-        limit = 3
+        limit = 2
     )
 
     return {