o y i'R@sdZddlZddlZddlZddlZddlZddlmZmZm Z m Z ddl m Z e eZzddlmZddlmZddlmZddlmZd ZWneyYd Zed YnwGd d d ZeZdS)zs Knowledge Base Service for Voice Bot Handles FAISS-based document search with S3 storage and business ID mapping. N)ListDictOptionalAny)Config)BedrockEmbeddings) PyPDFLoader)RecursiveCharacterTextSplitter)FAISSTFzMFAISS libraries not available. Knowledge base functionality will be disabled.c@s0eZdZdZddZddZd#dedeed efd d Z d$ded edeede d e e ee ff ddZde ee fd eefddZd%ded edeede d ef ddZd&deedeefddZd#dedeed efddZd#dedeed ee fddZd#de dedeefdd Zd efd!d"ZdS)'KnowledgeBaseServicezLManages FAISS-based document search with S3 storage and business ID mapping.cCsJtjot|_d|_d|_d|_d|_i|_|jr| dSt ddS)Nz6Knowledge base service disabled or FAISS not available) rZKNOWLEDGE_BASE_ENABLEDFAISS_AVAILABLEenabled s3_clientbedrock_clientbedrock_embeddings text_splittervector_store_cache_initialize_aws_clientsloggerinfoselfr"services/knowledge_base_service.py__init__s  zKnowledgeBaseService.__init__c CszEtjrtjstdd|_WdStjdtjtjtjd|_ tjdtjtjtjd|_ t tj |j d|_ ttjtjd |_td WdStyc}ztd |d|_WYd}~dSd}~ww) z*Initialize AWS clients for S3 and Bedrock.z8AWS credentials not configured. Knowledge base disabled.FNZs3) region_nameaws_access_key_idaws_secret_access_keyzbedrock-runtime)Z service_namerrr)Zmodel_idclient)Z chunk_sizeZ chunk_overlapz/Knowledge base service initialized successfullyz-Failed to initialize knowledge base service: )rZAWS_ACCESS_KEY_IDZAWS_SECRET_ACCESS_KEYrwarningr boto3rZ AWS_REGIONrrrZFAISS_EMBEDDING_MODELrr ZFAISS_CHUNK_SIZEZFAISS_CHUNK_OVERLAPrr Exceptionerror)rerrrr,s>  z,KnowledgeBaseService._initialize_aws_clientsN business_idbot_idreturnc Cs|js tddSz&|r|d|n|}||jvr+td|d|dWdS|r7d|d |d }nd|d }|jjtj|d }d |vrUtd |WdS| ||}|rp||j|<td |d|WdSg}|d D]}|d drzX|jj tj|dd} t j ddd} | | d| j} Wdn1swYt| } | } | D]}||jd<||jd<|d|jd<q|| t| Wqvty}ztd|dd|WYd}~qvd}~wwqv|std|WdS|j|}t||j}||j|<||||tdt|d|d|WdStyP}ztd|d|WYd}~dSd}~ww)z=Load documents for a specific business ID and bot ID from S3."Knowledge base service is disabledF_zDocuments for business and bot z already loadedTindexes//BucketZPrefixContentsz No documents found for business z*Loaded existing vector store for business Keyz.pdfr-r/)suffixdeleteZBodyNr$r%sourcezError processing PDF : z&No valid documents found for business zLoaded z document chunks for business z%Error loading documents for business ) r rrrrrlist_objects_v2rS3_BUCKET_NAME_load_vector_store_from_s3endswithZ get_objecttempfileZNamedTemporaryFilewritereadnamerloadmetadataextendosunlinkr!r"rZsplit_documentsr Zfrom_documentsr_save_vector_store_to_s3len)rr$r% cache_keyprefixresponse vector_storeZ documentsobjZpdf_objZ temp_fileZtemp_file_pathloaderZpdf_docsdocr#Z split_docsrrrload_documents_for_businessSs              z0KnowledgeBaseService.load_documents_for_businessquerytop_kc Cs|js tdgSza|r|d|n|}||jvr$|||s$gWS|j|}|s:td|d|gWS|j||d}g}|D]\} } || j| j t | dqEt dt |d|d d d |WSt y} ztd |d|d | gWYd } ~ Sd } ~ ww)z7Search documents for a specific business ID and bot ID.r'r(z#No vector store found for business r))k)contentr>scorezFound z relevant documents for query: N2...z'Error searching documents for business r4)r rrrrKgetZsimilarity_search_with_scoreappendZ page_contentr>floatrrCr!r") rr$rMr%rNrDrGZdocsresultsrJrQr#rrrsearch_documentss6      $z%KnowledgeBaseService.search_documentsdatac Csz[|did}|r!|dkr!|r!td||WSgd}|D],}||divrS|d|}|rS|dkrS|rStd|d||WSq'tdWd Styx}ztd t|WYd }~d Sd }~ww) zXExtract business ID from WebSocket data (X-BID from extra_headers) - From previous code.Z extra_headerszX-BIDz Not providedz+Found business ID in WebSocket data X-BID: )bidr$z business-idzx-bidz x-business-idz$Found business ID in WebSocket data r4z&No business ID found in WebSocket dataNz2Error extracting business ID from WebSocket data: )rTstriprrrr!r"str)rrYr$Zalternative_keyskeyrZr#rrr'extract_business_id_from_websocket_datas(   zsz.)r]rrPrQg@z[Score: z.2fz] dNrSz zGenerated context of z characters for business r)z (query: rRz...)z#Error getting context for business r4) r rXsortrCrUjoinrrr!r")rr$rMr%r`rWZ context_partsZ total_charsresultrPrQZcontent_with_scoreZremaining_charscontextr#rrrget_context_for_querys<  0z*KnowledgeBaseService.get_context_for_querycsrB|r#d|}||jvr!|j|=tdd|dSdSfdd|jD}|D]}|j|=q1tddS|jtddS) zClear the vector store cache.r(zCleared cache for business r)c g|] }|dr|qSr( startswith.0r]r$rr / z4KnowledgeBaseService.clear_cache..z'Cleared all cache entries for business z Cleared all knowledge base cacheN)rrrkeysclear)rr$r%rDkeys_to_remover]rrqr clear_cache%s   z KnowledgeBaseService.clear_cachec sF|jsdSz||rdd|d}ndd}|jjtj|d}d|vr|dD]}|ddrH|jjtj|ddtd |dd q)|r\d |}||j vr[|j |=nfd d |j D}|D]}|j |=qjtdd|WdSWdSt y} zt dd|d| WYd} ~ dSd} ~ ww)zAClear vector store files from S3 for a specific business and bot.Fr*r+r,r.r/).pkl.faissr0zDeleted z from S3r(crkrlrmrorqrrrrWrsz>KnowledgeBaseService.clear_s3_vector_store..z,Cleared all vector store files for business r)Tz,Error clearing S3 vector store for business r4N) r rr5rr6r8 delete_objectrrrrtr!r") rr$r%rErFrHrDrvr]r#rrqrclear_s3_vector_store7sF    z*KnowledgeBaseService.clear_s3_vector_storecCs$z|r d|d|d}nd|d}|jjtj|d}d|vrdd|dD}dd|dD}|r|rt}|dd }|dd } |jjtj||d d |jjtj| |d d z tjd ||j dd} Wn[t y} zOt d| t dz|jjtj|d|jjtj| dt dWnt y} zt d| WYd} ~ nd} ~ wwWYd} ~ WdWdSd} ~ wwt d|d|| WdWS1swYt d|d|WdSt y} zt dt| WYd} ~ dSd} ~ ww)z"Load existing vector store from S3r*r+r,r.cSg|] }|ddr|qS)r/rxr8rprHrrrrrtzCKnowledgeBaseService._load_vector_store_from_s3..cSr|)r/ryr}r~rrrrrurrr//vectorstore.pkl)r-r/Filename/vectorstore.faiss vectorstoreT) index_name folder_pathZ embeddingsZallow_dangerous_deserializationz2Failed to load vector store with standard method: zLClearing incompatible vector store files and will recreate from documents...r0z/Cleared incompatible vector store files from S3z%Could not delete incompatible files: Nz.Loaded existing vector store for business ID: and bot ID: z0No existing vector store found for business ID: z&Could not load existing vector store: )rr5rr6r9TemporaryDirectoryZ download_filer Z load_localrr!rrrrzr\)rr$r%rErFZ pkl_filesZ faiss_filestemp_dirpkl_key faiss_keyrGZ load_errorZ delete_errorr#rrrr7ds~       "1z/KnowledgeBaseService._load_vector_store_from_s3rGc CszftW}|jd|d|r"d|d|d}d|d|d}n d|d}d|d}|jj|dtj|d|jj|dtj|dtd|d |Wd Wd S1s_wYWd St y}zt d t |WYd }~d Sd }~ww) zSave vector store to S3r)rrr*r+rr)rr-r/z*Saved vector store to S3 for business ID: rNz!Error saving vector store to S3: ) r9rZ save_localrZ upload_filerr6rrr!r"r\)rrGr$r%rrrr#rrrrBs6   &"z-KnowledgeBaseService._save_vector_store_to_s3cCs|jo|jduS)z-Check if knowledge base service is available.N)r rrrrr is_availablesz!KnowledgeBaseService.is_available)N)NrL)Nr_)NN)__name__ __module__ __qualname____doc__rrr\rboolrKintrrrrXr^rjrwr{r7rBrrrrrr s '0a'$0 -M"r )rr r@ZjsonZloggingr9typingrrrrZconfigrZ getLoggerrrZ langchain_awsrZ$langchain_community.document_loadersrZlangchain.text_splitterr Z langchain_community.vectorstoresr r ImportErrorrr Zknowledge_base_servicerrrrs0        @