diff --git a/backend/score.py b/backend/score.py index 674c9044..a22d8b04 100644 --- a/backend/score.py +++ b/backend/score.py @@ -13,7 +13,7 @@ from src.graphDB_dataAccess import graphDBdataAccess from src.graph_query import get_graph_results,get_chunktext_results from src.chunkid_entities import get_entities_from_chunkids -from src.post_processing import create_vector_fulltext_indexes, create_entity_embedding, graph_cleanup +from src.post_processing import create_vector_fulltext_indexes, create_entity_embedding, graph_schema_consolidation from sse_starlette.sse import EventSourceResponse from src.communities import create_communities from src.neighbours import get_neighbour_nodes @@ -339,9 +339,9 @@ async def post_processing(uri=Form(), userName=Form(), password=Form(), database api_name = 'post_processing/create_entity_embedding' logging.info(f'Entity Embeddings created') - if "graph_cleanup" in tasks : - await asyncio.to_thread(graph_cleanup, graph) - api_name = 'post_processing/graph_cleanup' + if "graph_schema_consolidation" in tasks : + await asyncio.to_thread(graph_schema_consolidation, graph) + api_name = 'post_processing/graph_schema_consolidation' logging.info(f'Updated nodes and relationship labels') if "enable_communities" in tasks: diff --git a/backend/src/post_processing.py b/backend/src/post_processing.py index cc2596c8..8b79f93b 100644 --- a/backend/src/post_processing.py +++ b/backend/src/post_processing.py @@ -194,7 +194,7 @@ def update_embeddings(rows, graph): """ return graph.query(query,params={'rows':rows}) -def graph_cleanup(graph): +def graph_schema_consolidation(graph): nodes_and_relations = get_labels_and_relationtypes(graph) logging.info(f"nodes_and_relations in existing graph : {nodes_and_relations}") node_labels = [] diff --git a/frontend/src/components/Content.tsx b/frontend/src/components/Content.tsx index 672eabab..8967c7df 100644 --- a/frontend/src/components/Content.tsx +++ b/frontend/src/components/Content.tsx @@ -44,6 +44,7 @@ import PostProcessingToast from './Popups/GraphEnhancementDialog/PostProcessingC import { getChunkText } from '../services/getChunkText'; import ChunkPopUp from './Popups/ChunkPopUp'; import { isExpired, isFileReadyToProcess } from '../utils/Utils'; +import { useHasSelections } from '../hooks/useHasSelections'; const ConfirmationDialog = lazy(() => import('./Popups/LargeFilePopUp/ConfirmationDialog')); @@ -105,6 +106,7 @@ const Content: React.FC = ({ ); const [showDeletePopUp, setshowDeletePopUp] = useState(false); const [deleteLoading, setdeleteLoading] = useState(false); + const hasSelections = useHasSelections(selectedNodes, selectedRels); const { updateStatusForLargeFiles } = useServerSideEvent( (inMinutes, time, fileName) => { @@ -139,16 +141,16 @@ const Content: React.FC = ({ 0 || selectedRels.length > 0} + isSchema={hasSelections} /> ); try { const payload = isGdsActive - ? selectedNodes.length > 0 || selectedRels.length > 0 - ? postProcessingTasks.filter((task) => task !== 'graph_cleanup') + ? hasSelections + ? postProcessingTasks.filter((task) => task !== 'graph_schema_consolidation') : postProcessingTasks - : selectedNodes.length > 0 || selectedRels.length > 0 - ? postProcessingTasks.filter((task) => task !== 'graph_cleanup' && task !== 'enable_communities') + : hasSelections + ? postProcessingTasks.filter((task) => task !== 'graph_schema_consolidation' && task !== 'enable_communities') : postProcessingTasks.filter((task) => task !== 'enable_communities'); const response = await postProcessing(userCredentials as UserCredentials, payload); if (response.data.status === 'Success') { @@ -379,7 +381,7 @@ const Content: React.FC = ({ const addFilesToQueue = async (remainingFiles: CustomFile[]) => { if (!remainingFiles.length) { showNormalToast( - 0 || selectedRels.length > 0} /> + ); try { const response = await postProcessing(userCredentials as UserCredentials, postProcessingTasks); @@ -704,7 +706,7 @@ const Content: React.FC = ({ const selectedRows = childRef.current?.getSelectedRows(); if (selectedRows?.length) { const expiredFilesExists = selectedRows.some( - (c) => c.status !== 'Ready to Reprocess' && isExpired(c?.createdAt as Date) + (c) => c.status !== 'Ready to Reprocess' && isExpired(c?.createdAt as Date ?? new Date()) ); const largeFileExists = selectedRows.some( (c) => isFileReadyToProcess(c, true) && typeof c.size === 'number' && c.size > largeFileSize @@ -718,7 +720,7 @@ const Content: React.FC = ({ } } else if (filesData.length) { const expiredFileExists = filesData.some( - (c) => isExpired(c.createdAt as Date) + (c) => isExpired(c?.createdAt as Date) ); const largeFileExists = filesData.some( (c) => isFileReadyToProcess(c, true) && typeof c.size === 'number' && c.size > largeFileSize @@ -847,17 +849,17 @@ const Content: React.FC = ({ />
- {selectedNodes.length === 0 || selectedRels.length === 0 ? ( + {!hasSelections ? ( ) : ( )}
- {selectedNodes.length > 0 || selectedRels.length > 0 ? ( + {hasSelections? ( - {(!selectedNodes.length || !selectedRels.length) && 'Empty'} Graph Schema configured - {selectedNodes.length || selectedRels.length + {(hasSelections)} Graph Schema configured + {hasSelections ? `(${selectedNodes.length} Labels + ${selectedRels.length} Rel Types)` : ''} diff --git a/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/EntityExtractionSetting.tsx b/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/EntityExtractionSetting.tsx index 4785952a..c05bd085 100644 --- a/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/EntityExtractionSetting.tsx +++ b/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/EntityExtractionSetting.tsx @@ -9,6 +9,7 @@ import { OptionType, schema, UserCredentials } from '../../../../types'; import { getNodeLabelsAndRelTypes } from '../../../../services/GetNodeLabelsRelTypes'; import { tokens } from '@neo4j-ndl/base'; import { showNormalToast } from '../../../../utils/toasts'; +import { useHasSelections } from '../../../../hooks/useHasSelections'; export default function EntityExtractionSetting({ view, @@ -39,6 +40,7 @@ export default function EntityExtractionSetting({ const { userCredentials } = useCredentials(); const [loading, setLoading] = useState(false); const isTablet = useMediaQuery(`(min-width:${breakpoints.xs}) and (max-width: ${breakpoints.lg})`); + const hasSelections = useHasSelections(selectedNodes, selectedRels); const removeNodesAndRels = (nodelabels: string[], relationshipTypes: string[]) => { const labelsToRemoveSet = new Set(nodelabels); const relationshipLabelsToremoveSet = new Set(relationshipTypes); @@ -346,7 +348,7 @@ export default function EntityExtractionSetting({ placement='top' onClick={handleClear} label='Clear Graph Settings' - disabled={selectedNodes.length === 0 || selectedRels.length === 0} + disabled={!hasSelections} > {buttonCaptions.clearSettings} @@ -356,7 +358,7 @@ export default function EntityExtractionSetting({ placement='top' onClick={handleApply} label='Apply Graph Settings' - disabled={selectedNodes.length === 0 || selectedRels.length === 0} + disabled={!hasSelections} > {buttonCaptions.applyGraphSchema} diff --git a/frontend/src/components/Popups/GraphEnhancementDialog/PostProcessingCheckList/index.tsx b/frontend/src/components/Popups/GraphEnhancementDialog/PostProcessingCheckList/index.tsx index c55d47ea..d9e7175c 100644 --- a/frontend/src/components/Popups/GraphEnhancementDialog/PostProcessingCheckList/index.tsx +++ b/frontend/src/components/Popups/GraphEnhancementDialog/PostProcessingCheckList/index.tsx @@ -21,17 +21,17 @@ export default function PostProcessingCheckList() { {POST_PROCESSING_JOBS.map((job, idx) => { const isGraphCleanupDisabled = - job.title === 'graph_cleanup' + job.title === 'graph_schema_consolidation' ? !(selectedNodes.length === 0 && selectedRels.length === 0) : false; const isDisabled = job.title === 'enable_communities' ? !isGdsActive - : job.title === 'graph_cleanup' + : job.title === 'graph_schema_consolidation' ? isGraphCleanupDisabled : false; const isChecked = - job.title === 'graph_cleanup' + job.title === 'graph_schema_consolidation' ? !isGraphCleanupDisabled && postProcessingTasks.includes(job.title) : job.title === 'enable_communities' ? isGdsActive && postProcessingTasks.includes(job.title) diff --git a/frontend/src/context/UsersFiles.tsx b/frontend/src/context/UsersFiles.tsx index d8c805aa..1487e35a 100644 --- a/frontend/src/context/UsersFiles.tsx +++ b/frontend/src/context/UsersFiles.tsx @@ -43,7 +43,7 @@ const FileContextProvider: FC = ({ children }) => { 'enable_hybrid_search_and_fulltext_search_in_bloom', 'materialize_entity_similarities', 'enable_communities', - 'graph_cleanup', + 'graph_schema_consolidation', ]); const [processedCount, setProcessedCount] = useState(0); const [postProcessingVal, setPostProcessingVal] = useState(false); diff --git a/frontend/src/hooks/useHasSelections.tsx b/frontend/src/hooks/useHasSelections.tsx new file mode 100644 index 00000000..ff06b291 --- /dev/null +++ b/frontend/src/hooks/useHasSelections.tsx @@ -0,0 +1,7 @@ +import { useMemo } from "react"; + +import { OptionType } from "../types"; +export const useHasSelections = (selectedNodes: readonly OptionType[], selectedRels: readonly OptionType[]) => { + const hasSelections = useMemo(()=> selectedNodes.length> 0 || selectedRels.length > 0,[selectedNodes, selectedRels]); + return hasSelections; +} diff --git a/frontend/src/utils/Constants.ts b/frontend/src/utils/Constants.ts index 8763374a..f894bc67 100644 --- a/frontend/src/utils/Constants.ts +++ b/frontend/src/utils/Constants.ts @@ -18,6 +18,7 @@ export const llms = 'openai_gpt_4o_mini', 'gemini_1.5_pro', 'gemini_1.5_flash', + 'gemini_2.0_flash', 'diffbot', 'azure_ai_gpt_35', 'azure_ai_gpt_4o', @@ -25,6 +26,7 @@ export const llms = 'groq_llama3_70b', 'anthropic_claude_3_5_sonnet', 'fireworks_llama_v3p2_90b', + 'fireworks_qwen72b_instruct', 'bedrock_claude_3_5_sonnet', ]; @@ -179,7 +181,7 @@ export const buttonCaptions = { provideAdditionalInstructions: 'Provide Additional Instructions for Entity Extractions', analyzeInstructions: 'Analyze Instructions', helpInstructions: - 'Provide specific instructions for entity extraction, such as focusing on key topics or excluding non-entity data like dates or revenues.', + 'Provide specific instructions for entity extraction, such as focusing on the key topics.', }; export const POST_PROCESSING_JOBS: { title: string; description: string }[] = [ @@ -209,8 +211,8 @@ export const POST_PROCESSING_JOBS: { title: string; description: string }[] = [ description: 'Enable community creation across entities to use GraphRAG capabilities both local and global search.', }, { - title: 'graph_cleanup', - description: 'consolidate node-labels and rel-types and then update the graph ', + title: 'graph_schema_consolidation', + description: 'This option uses the LLM for large graph schemas to consolidate many node labels and relationship types into fewer, more relevant ones and apply it to the extracted and existing graph', }, ]; export const RETRY_OPIONS = [