Dev to staging

README.md

@@ -18,7 +18,7 @@ Upload your files from local machine, GCS or S3 bucket or from web sources, choo
 - **Knowledge Graph Creation**: Transform unstructured data into structured knowledge graphs using LLMs.
 - **Providing Schema**: Provide your own custom schema or use existing schema in settings to generate graph.
 - **View Graph**: View graph for a particular source or multiple sources at a time in Bloom.
-- **Chat with Data**: Interact with your data in a Neo4j database through conversational queries, also retrieve metadata about the source of response to your queries. 
+- **Chat with Data**: Interact with your data in a Neo4j database through conversational queries, also retrieve metadata about the source of response to your queries.For a dedicated chat interface, access the standalone chat application at: [Chat-Only](https://dev-frontend-dcavk67s4a-uc.a.run.app/chat-only). This link provides a focused chat experience for querying your data.
 
 ## Getting started
 
@@ -37,12 +37,6 @@ EX:
 VITE_LLM_MODELS_PROD="openai_gpt_4o,openai_gpt_4o_mini,diffbot,gemini_1.5_flash"
 ```
 
-In your root folder, create a .env file with your OPENAI and DIFFBOT keys (if you want to use both):
-```env
-OPENAI_API_KEY="your-openai-key"
-DIFFBOT_API_KEY="your-diffbot-key"
-```
-
 if you only want OpenAI:
 ```env
 VITE_LLM_MODELS_PROD="diffbot,openai-gpt-3.5,openai-gpt-4o"
@@ -102,9 +96,13 @@ Alternatively, you can run the backend and frontend separately:
     ```
 
 - For the backend:
-1. Create the backend/.env file by copy/pasting the backend/example.env.
-2. Change values as needed
-3.
+1. Create the backend/.env file by copy/pasting the backend/example.env. To streamline the initial setup and testing of the application, you can preconfigure user credentials directly within the .env file. This bypasses the login dialog and allows you to immediately connect with a predefined user.
+   - **NEO4J_URI**:
+   - **NEO4J_USERNAME**:
+   - **NEO4J_PASSWORD**:
+   - **NEO4J_DATABASE**:
+3. Change values as needed
+4.
     ```bash
     cd backend
     python -m venv envName
@@ -202,7 +200,7 @@ VITE_BACKEND_API_URL=${VITE_BACKEND_API_URL-backendurl}
 
 
 ## Usage
-1. Connect to Neo4j Aura Instance which can be both AURA DS or AURA DB by passing URI and password or using Neo4j credentials file.
+1. Connect to Neo4j Aura Instance which can be both AURA DS or AURA DB by passing URI and password through Backend env, fill using login dialog or drag and drop the Neo4j credentials file.
 2. To differntiate we have added different icons. For AURA DB we have a database icon and for AURA DS we have scientific molecule icon right under Neo4j Connection details label.
 3. Choose your source from a list of Unstructured sources to create graph.
 4. Change the LLM (if required) from drop down, which will be used to generate graph.

backend/README.md

@@ -50,11 +50,11 @@ http://127.0.0.1:8000/redocs for ReDoc.
 
 ## Configuration
 
-Update the environment variable in `.env` file.
+Update the environment variable in `.env` file. Refer example.env in backend folder for more config.
 
-`OPENAI_API_KEY`: Open AI key to use LLM
+`OPENAI_API_KEY`: Open AI key to use incase of openai embeddings
 
-`DIFFBOT_API_KEY` : Diffbot API key to use DiffbotGraphTransformer
+`EMBEDDING_MODEL` : "all-MiniLM-L6-v2" or "openai" or "vertexai"
 
 `NEO4J_URI` : Neo4j URL
 

backend/example.env

@@ -1,6 +1,5 @@
 OPENAI_API_KEY = ""
-DIFFBOT_API_KEY = ""
-GROQ_API_KEY = ""
+#EMBEDDING_MODEL can be openai or vertexai or by default all-MiniLM-L6-v2
 EMBEDDING_MODEL = "all-MiniLM-L6-v2"
 RAGAS_EMBEDDING_MODEL = "openai"
 IS_EMBEDDING = "true"

backend/src/graphDB_dataAccess.py

@@ -362,7 +362,7 @@ def get_duplicate_nodes_list(self):
         score_value = float(os.environ.get('DUPLICATE_SCORE_VALUE'))
         text_distance = int(os.environ.get('DUPLICATE_TEXT_DISTANCE'))
         query_duplicate_nodes = """
-                MATCH (n:!Chunk&!Session&!Document&!`__Community__`&!`__Entity__`) with n 
+                MATCH (n:!Chunk&!Session&!Document&!`__Community__`) with n 
                 WHERE n.embedding is not null and n.id is not null // and size(toString(n.id)) > 3
                 WITH n ORDER BY count {{ (n)--() }} DESC, size(toString(n.id)) DESC // updated
                 WITH collect(n) as nodes

backend/src/make_relationships.py

@@ -68,12 +68,12 @@ def update_embedding_create_vector_index(graph, chunkId_chunkDoc_list, file_name
             #                 "embeddings" : embeddings_arr
             #             }
             #             )
-            # logging.info('create vector index on chunk embedding')
-            result = graph.query("SHOW INDEXES YIELD * WHERE labelsOrTypes = ['__Chunk__'] and name = 'vector'")
+            logging.info('create vector index on chunk embedding')
+            # result = graph.query("SHOW INDEXES YIELD * WHERE labelsOrTypes = ['Chunk'] and name = 'vector'")
             vector_index = graph.query("SHOW INDEXES YIELD * WHERE labelsOrTypes = ['Chunk'] and type = 'VECTOR' AND name = 'vector' return options")
-            if result:
-                logging.info(f"vector index dropped for 'Chunk'")
-                graph.query("DROP INDEX vector IF EXISTS;")
+            # if result:
+            #     logging.info(f"vector index dropped for 'Chunk'")
+            #     graph.query("DROP INDEX vector IF EXISTS;")
 
             if len(vector_index) == 0:
                 logging.info(f'vector index is not exist, will create in next query')

backend/src/post_processing.py

@@ -124,6 +124,10 @@ def create_fulltext(driver,type):
 
 def create_vector_fulltext_indexes(uri, username, password, database):
     types = ["entities", "hybrid"]
+    embedding_model = os.getenv('EMBEDDING_MODEL')
+    embeddings, dimension = load_embedding_model(embedding_model)
+    if not dimension:
+        dimension = CHUNK_VECTOR_EMBEDDING_DIMENSION
     logging.info("Starting the process of creating full-text indexes.")
 
     try:
@@ -144,7 +148,7 @@ def create_vector_fulltext_indexes(uri, username, password, database):
 
     try:
         logging.info(f"Creating a vector index for type '{CHUNK_VECTOR_INDEX_NAME}'.")
-        create_vector_index(driver, CHUNK_VECTOR_INDEX_NAME,CHUNK_VECTOR_EMBEDDING_DIMENSION)
+        create_vector_index(driver, CHUNK_VECTOR_INDEX_NAME,dimension)
         logging.info("Vector index for chunk created successfully.")
     except Exception as e:
         logging.error(f"Failed to create vector index for '{CHUNK_VECTOR_INDEX_NAME}': {e}")

frontend/src/components/ChatBot/ChatInfoModal.tsx

@@ -96,16 +96,16 @@ const ChatInfoModal: React.FC<chatInfoMessage> = ({
   const [enableReference, toggleReferenceVisibility] = useReducer((state: boolean) => !state, false);
   const textAreaRef = useRef<HTMLTextAreaElement>(null);
   const [isAdditionalMetricsEnabled, setIsAdditionalMetricsEnabled] = useState<boolean | null>(
-    multiModelMetrics.length > 0 && Object.keys(multiModelMetrics[0]).length > 3
+    multiModelMetrics.length > 0 && Object.keys(multiModelMetrics[0]).length > 4
       ? true
-      : multiModelMetrics.length > 0 && Object.keys(multiModelMetrics[0]).length <= 3
+      : multiModelMetrics.length > 0 && Object.keys(multiModelMetrics[0]).length <= 4
       ? false
       : null
   );
   const [isAdditionalMetricsWithSingleMode, setIsAdditionalMetricsWithSingleMode] = useState<boolean | null>(
-    metricDetails != undefined && Object.keys(metricDetails).length > 2
+    metricDetails != undefined && Object.keys(metricDetails).length > 3
       ? true
-      : metricDetails != undefined && Object.keys(metricDetails).length <= 2
+      : metricDetails != undefined && Object.keys(metricDetails).length <= 3
       ? false
       : null
   );

frontend/src/components/ChatBot/Chatbot.tsx

@@ -578,7 +578,7 @@ const Chatbot: FC<ChatbotProps> = (props) => {
           }}
           onClose={() => setShowInfoModal(false)}
           isOpen={showInfoModal}
-          size={activeChat?.currentMode === chatModeLables['entity search+vector'] ? 'large' : 'medium'}
+          size={'large'}
         >
           <div style={{ display: 'flex', justifyContent: 'flex-end', alignItems: 'center' }}>
             <IconButton

frontend/src/components/ChatBot/MultiModeMetrics.tsx

@@ -6,7 +6,7 @@ import {
   getPaginationRowModel,
 } from '@tanstack/react-table';
 import { capitalize } from '../../utils/Utils';
-import { useContext, useEffect, useMemo, useRef } from 'react';
+import { useContext, useMemo, useRef } from 'react';
 import { Banner, Box, DataGrid, DataGridComponents, Flex, IconButton, Popover, Typography } from '@neo4j-ndl/react';
 import { multimodelmetric } from '../../types';
 import { ThemeWrapperContext } from '../../context/ThemeWrapper';
@@ -47,6 +47,7 @@ export default function MultiModeMetrics({
         },
         header: () => <span>Mode</span>,
         footer: (info) => info.column.id,
+        maxSize: 150,
       }),
       columnHelper.accessor((row) => row.answer_relevancy as number, {
         id: 'Answer Relevancy',
@@ -74,6 +75,7 @@ export default function MultiModeMetrics({
             </Popover>
           </Flex>
         ),
+        maxSize: 150,
       }),
       columnHelper.accessor((row) => row.faithfulness as number, {
         id: 'Faithfullness',
@@ -101,6 +103,7 @@ export default function MultiModeMetrics({
             </Popover>
           </Flex>
         ),
+        maxSize: 150,
       }),
       columnHelper.accessor((row) => row.context_entity_recall as number, {
         id: 'Entity Recall Score',
@@ -128,6 +131,7 @@ export default function MultiModeMetrics({
             </Popover>
           </Flex>
         ),
+        maxSize: 150,
       }),
       columnHelper.accessor((row) => row.semantic_score as number, {
         id: 'Semantic Score',
@@ -155,6 +159,7 @@ export default function MultiModeMetrics({
             </Popover>
           </Flex>
         ),
+        maxSize: 150,
       }),
       columnHelper.accessor((row) => row.rouge_score as number, {
         id: 'Rouge Score',
@@ -182,30 +187,136 @@ export default function MultiModeMetrics({
             </Popover>
           </Flex>
         ),
+        maxSize: 150,
       }),
     ],
     []
   );
-  const table = useReactTable({
-    data,
-    columns,
-    getCoreRowModel: getCoreRowModel(),
-    getFilteredRowModel: getFilteredRowModel(),
-    getPaginationRowModel: getPaginationRowModel(),
-    enableGlobalFilter: false,
-    autoResetPageIndex: false,
-    enableColumnResizing: true,
-    enableRowSelection: true,
-    enableMultiRowSelection: true,
-    enableSorting: false,
-  });
-  useEffect(() => {
-    if (isWithAdditionalMetrics === false) {
-      table.setColumnVisibility({ 'Semantic Score': false, 'Rouge Score': false });
-    } else {
-      table.resetColumnVisibility(true);
-    }
-  }, [isWithAdditionalMetrics, table]);
+
+  const columnswithoutSemanticAndRougeScores = useMemo(
+    () => [
+      columnHelper.accessor((row) => row.mode, {
+        id: 'Mode',
+        cell: (info) => {
+          const metric = info.getValue();
+          const capitilizedMetric = metric.includes('_')
+            ? metric
+                .split('_')
+                .map((w) => capitalize(w))
+                .join(' ')
+            : capitalize(metric);
+          return (
+            <div className='textellipsis'>
+              <span title={metric}>{capitilizedMetric}</span>
+            </div>
+          );
+        },
+        header: () => <span>Mode</span>,
+        footer: (info) => info.column.id,
+        maxSize: 150,
+      }),
+      columnHelper.accessor((row) => row.answer_relevancy as number, {
+        id: 'Answer Relevancy',
+        cell: (info) => {
+          const value = isNaN(info.getValue()) ? 'N.A' : info.getValue()?.toFixed(2);
+          if (value === 'N.A') {
+            return <NotAvailableMetric />;
+          }
+          return <Typography variant='body-medium'>{value}</Typography>;
+        },
+        header: () => (
+          <Flex flexDirection='row' alignItems='center'>
+            <span>Relevancy</span>
+            <Popover placement='top-middle-bottom-middle' hasAnchorPortal={true}>
+              <Popover.Trigger hasButtonWrapper>
+                <IconButton size='small' isClean ariaLabel='infoicon'>
+                  <InformationCircleIconOutline />
+                </IconButton>
+              </Popover.Trigger>
+              <Popover.Content className='p-2'>
+                <Typography variant='body-small'>
+                  Determines How well the answer addresses the user's question.
+                </Typography>
+              </Popover.Content>
+            </Popover>
+          </Flex>
+        ),
+      }),
+      columnHelper.accessor((row) => row.faithfulness as number, {
+        id: 'Faithfullness',
+        cell: (info) => {
+          const value = isNaN(info.getValue()) ? 'N.A' : info.getValue()?.toFixed(2);
+          if (value === 'N.A') {
+            return <NotAvailableMetric />;
+          }
+          return <Typography variant='body-medium'>{value}</Typography>;
+        },
+        header: () => (
+          <Flex flexDirection='row' alignItems='center'>
+            <span>Faithful</span>
+            <Popover placement='top-middle-bottom-middle' hasAnchorPortal={true}>
+              <Popover.Trigger hasButtonWrapper>
+                <IconButton size='small' isClean ariaLabel='infoicon'>
+                  <InformationCircleIconOutline />
+                </IconButton>
+              </Popover.Trigger>
+              <Popover.Content className='p-2'>
+                <Typography variant='body-small'>
+                  Determines How accurately the answer reflects the provided information.
+                </Typography>
+              </Popover.Content>
+            </Popover>
+          </Flex>
+        ),
+      }),
+      columnHelper.accessor((row) => row.context_entity_recall as number, {
+        id: 'Entity Recall Score',
+        cell: (info) => {
+          const value = isNaN(info.getValue()) ? 'N.A' : info.getValue()?.toFixed(2);
+          if (value === 'N.A') {
+            return <NotAvailableMetric />;
+          }
+          return <Typography variant='body-medium'>{value}</Typography>;
+        },
+        header: () => (
+          <Flex flexDirection='row' alignItems='center'>
+            <span>Context</span>
+            <Popover placement='top-middle-bottom-middle' hasAnchorPortal={true}>
+              <Popover.Trigger hasButtonWrapper>
+                <IconButton size='small' isClean ariaLabel='infoicon'>
+                  <InformationCircleIconOutline />
+                </IconButton>
+              </Popover.Trigger>
+              <Popover.Content className='p-2'>
+                <Typography variant='body-small'>
+                  Determines the recall of entities present in both generated answer and retrieved contexts.
+                </Typography>
+              </Popover.Content>
+            </Popover>
+          </Flex>
+        ),
+      }),
+    ],
+    []
+  );
+  const config = useMemo(
+    () => ({
+      data,
+      columns: !isWithAdditionalMetrics ? columnswithoutSemanticAndRougeScores : columns,
+      getCoreRowModel: getCoreRowModel(),
+      getFilteredRowModel: getFilteredRowModel(),
+      getPaginationRowModel: getPaginationRowModel(),
+      enableGlobalFilter: false,
+      autoResetPageIndex: false,
+      enableColumnResizing: true,
+      enableRowSelection: true,
+      enableMultiRowSelection: true,
+      enableSorting: false,
+    }),
+    [isWithAdditionalMetrics]
+  );
+  const table = useReactTable(config);
+
 
   return (
     <Box>
@@ -226,7 +337,7 @@ export default function MultiModeMetrics({
             }}
             isAutoResizingColumns={true}
             isLoading={metricsLoading}
-            rootProps={{ className: isWithAdditionalMetrics === false ? '!w-[465px]' : 'auto' }}
+            // rootProps={{ className: isWithAdditionalMetrics === false ? '!w-[465px]' : 'auto' }}
             components={{
               Body: () => (
                 <DataGridComponents.Body

frontend/src/components/FileTable.tsx

@@ -120,6 +120,7 @@ const FileTable = forwardRef<ChildRef, FileTableProps>((props, ref) => {
           return (
             <div className='px-1'>
               <Checkbox
+                ariaLabel='row-selection'
                 isChecked={row.getIsSelected()}
                 isDisabled={
                   !row.getCanSelect() ||

frontend/src/components/Layout/DrawerDropzone.tsx

@@ -41,9 +41,15 @@ const DrawerDropzone: React.FC<DrawerProps> = ({
 
   return (
     <div className='flex min-h-[calc(-58px+100vh)] relative'>
-      <Drawer isExpanded={isExpanded} position='left' type='push' isCloseable={false}>
+      <Drawer
+        isExpanded={isExpanded}
+        position='left'
+        type='push'
+        isCloseable={false}
+        htmlAttributes={{ style: { height: 'intial' } }}
+      >
         {!isReadOnlyUser ? (
-          <Drawer.Body className={`!overflow-hidden !w-[294px]`} htmlAttributes={{ style: { height: 'intial' } }}>
+          <Drawer.Body className={`!overflow-hidden !w-[294px]`}>
             {alertState.showAlert && (
               <CustomAlert
                 severity={alertState.alertType}

frontend/src/components/UI/ButtonWithToolTip.tsx

@@ -33,7 +33,7 @@ const ButtonWithToolTip = ({
   const [isHovered, setIsHovered] = useState<boolean>(false);
   return (
     <Tooltip placement={placement} type='simple'>
-      <Tooltip.Trigger>
+      <Tooltip.Trigger hasButtonWrapper>
         <Button
           size={size}
           onClick={onClick}