diff --git a/README.md b/README.md index b48711f2..6a733488 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ Send the request to the API: ``` curl -X POST -H "Content-Type: application/json" -d '{ "payload": { - "user_id": "681", + "user_id": "97980cfea0067", "data": [".data/3ZCCCW.pdf"], "test_set": "sample", "params": ["chunk_size"], @@ -217,7 +217,7 @@ After that, you can run the RAG test manager from your command line. python rag_test_manager.py \ --file ".data" \ --test_set "example_data/test_set.json" \ - --user_id "666" \ + --user_id "97980cfea0067" \ --params "chunk_size" "search_type" \ --metadata "example_data/metadata.json" \ --retriever_type "single_document_context" @@ -226,3 +226,6 @@ After that, you can run the RAG test manager from your command line. Examples of metadata structure and test set are in the folder "example_data" + + + diff --git a/level_3/.env.template b/level_3/.env.template index 8ca7daed..f3e42390 100644 --- a/level_3/.env.template +++ b/level_3/.env.template @@ -6,4 +6,5 @@ POSTGRES_USER = bla POSTGRES_PASSWORD = bla POSTGRES_DB = bubu POSTGRES_HOST = localhost -POSTGRES_HOST_DOCKER = postgres \ No newline at end of file +POSTGRES_HOST_DOCKER = postgres +SEGMENT_KEY = Etl4WJwzOkeDPAjaOXOMgyU16hO7mV7B \ No newline at end of file diff --git a/level_3/poetry.lock b/level_3/poetry.lock index b1729ae3..15e69a46 100644 --- a/level_3/poetry.lock +++ b/level_3/poetry.lock @@ -2147,13 +2147,13 @@ six = "*" [[package]] name = "langsmith" -version = "0.0.53" +version = "0.0.54" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langsmith-0.0.53-py3-none-any.whl", hash = "sha256:a090b1c7d7968fb8d2476ddd608a5171f0e812a82b1bca29ca136cdea375a74e"}, - {file = "langsmith-0.0.53.tar.gz", hash = "sha256:a426a1d39843207a5dd3d72787b5304376541eb818509ee7909bbb696b072488"}, + {file = "langsmith-0.0.54-py3-none-any.whl", hash = "sha256:55eca5967cadb661a49ad32aecda48a824fadef202ca384575209a9d6f823b74"}, + {file = "langsmith-0.0.54.tar.gz", hash = "sha256:76c8e34b4d10ad93541107138089635829f9d60601a7f6bddf5ba582d178e521"}, ] [package.dependencies] @@ -2576,6 +2576,17 @@ files = [ {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, ] +[[package]] +name = "monotonic" +version = "1.6" +description = "An implementation of time.monotonic() for Python 2 & < 3.3" +optional = false +python-versions = "*" +files = [ + {file = "monotonic-1.6-py2.py3-none-any.whl", hash = "sha256:68687e19a14f11f26d140dd5c86f3dba4bf5df58003000ed467e0e2a69bca96c"}, + {file = "monotonic-1.6.tar.gz", hash = "sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7"}, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -4572,6 +4583,26 @@ dev = ["flake8", "mypy", "pycodestyle", "typing_extensions"] doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-panels (>=0.5.2)", "sphinx-tabs"] test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +[[package]] +name = "segment-analytics-python" +version = "2.2.3" +description = "The hassle-free way to integrate analytics into any python application." +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "segment-analytics-python-2.2.3.tar.gz", hash = "sha256:0df5908e3df74b4482f33392fdd450df4c8351bf54974376fbe6bf33b0700865"}, + {file = "segment_analytics_python-2.2.3-py2.py3-none-any.whl", hash = "sha256:06cc3d8e79103f02c3878ec66cb66152415473d0d2a142b98a0ee18da972e109"}, +] + +[package.dependencies] +backoff = ">=2.1,<3.0" +monotonic = ">=1.5,<2.0" +python-dateutil = ">=2.2,<3.0" +requests = ">=2.7,<3.0" + +[package.extras] +test = ["flake8 (==3.7.9)", "mock (==2.0.0)", "pylint (==2.8.0)"] + [[package]] name = "semver" version = "3.0.2" @@ -6209,4 +6240,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "9a121700fa6a5d396844af0958e72b437b5819cf30970f4176e002709feca667" +content-hash = "1700f1cab54ed107d299a47031ad53c58e5e72e3791b9113652fb4ff7854a91a" diff --git a/level_3/pyproject.toml b/level_3/pyproject.toml index a8bd4568..60eb074b 100644 --- a/level_3/pyproject.toml +++ b/level_3/pyproject.toml @@ -40,6 +40,7 @@ dash = "^2.14.0" unstructured = {extras = ["pdf"], version = "^0.10.23"} sentence-transformers = "2.2.2" torch = "2.0.*" +segment-analytics-python = "^2.2.3" diff --git a/level_3/rag_test_manager.py b/level_3/rag_test_manager.py index 2eb9dd9d..108b98d0 100644 --- a/level_3/rag_test_manager.py +++ b/level_3/rag_test_manager.py @@ -32,6 +32,8 @@ from models.operation import Operation from models.docs import DocsModel +import segment.analytics as analytics + load_dotenv() import ast import tracemalloc @@ -55,8 +57,16 @@ from database.database import AsyncSessionLocal openai.api_key = os.getenv("OPENAI_API_KEY", "") +analytics.write_key = os.getenv("SEGMENT_KEY", "") + + +def on_error(error, items): + print("An error occurred:", error) +analytics.debug = True +analytics.on_error = on_error + async def retrieve_latest_test_case(session, user_id, memory_id): try: # Use await with session.execute() and row.fetchone() or row.all() for async query execution @@ -476,6 +486,10 @@ async def start_test( await add_entity( session, TestSet(id=test_set_id, user_id=user_id, content=str(test_set)) ) + analytics.track(user_id, 'TestSet', { + 'id': test_set_id, + 'content': str(test_set) + }) if params is None: data_format = data_format_route( @@ -521,6 +535,15 @@ async def start_test( test_set_id=test_set_id, ), ) + analytics.track(user_id, 'Operation', { + 'id': job_id, + 'operation_params': str(test_params), + 'number_of_files': count_files_in_data_folder(), + 'operation_status': "RUNNING", + 'operation_type': retriever_type, + 'test_set_id': test_set_id, + }) + doc_names = get_document_names(data) for doc in doc_names: @@ -697,6 +720,20 @@ async def run_generate_test_set(test_id): test_params=str(chunk), # Add params to the database table ), ) + analytics.track(user_id, 'TestOutput', { + 'test_set_id': test_set_id, + 'operation_id': job_id, + 'set_id' : str(uuid.uuid4()), + 'test_results' : result["success"], + 'test_score' : str(result["score"]), + 'test_metric_name' : result["metric_name"], + 'test_query' : result["query"], + 'test_output' : result["output"], + 'test_expected_output' : str(["expected_output"]), + 'test_context' : result["context"][0], + 'test_params' : str(chunk), + }) + analytics.flush() await update_entity(session, Operation, job_id, "COMPLETED") @@ -704,49 +741,6 @@ async def run_generate_test_set(test_id): async def main(): - # metadata = { - # "version": "1.0", - # "agreement_id": "AG123456", - # "privacy_policy": "https://example.com/privacy", - # "terms_of_service": "https://example.com/terms", - # "format": "json", - # "schema_version": "1.1", - # "checksum": "a1b2c3d4e5f6", - # "owner": "John Doe", - # "license": "MIT", - # "validity_start": "2023-08-01", - # "validity_end": "2024-07-31", - # } - # - # test_set = [ - # { - # "question": "Who is the main character in 'The Call of the Wild'?", - # "answer": "Buck", - # }, - # {"question": "Who wrote 'The Call of the Wild'?", "answer": "Jack London"}, - # { - # "question": "Where does Buck live at the start of the book?", - # "answer": "In the Santa Clara Valley, at Judge Miller’s place.", - # }, - # { - # "question": "Why is Buck kidnapped?", - # "answer": "He is kidnapped to be sold as a sled dog in the Yukon during the Klondike Gold Rush.", - # }, - # { - # "question": "How does Buck become the leader of the sled dog team?", - # "answer": "Buck becomes the leader after defeating the original leader, Spitz, in a fight.", - # }, - # ] - # "https://www.ibiblio.org/ebooks/London/Call%20of%20Wild.pdf" - # # http://public-library.uk/ebooks/59/83.pdf - # result = await start_test( - # [".data/3ZCCCW.pdf"], - # test_set=test_set, - # user_id="677", - # params=["chunk_size", "search_type"], - # metadata=metadata, - # retriever_type="single_document_context", - # ) parser = argparse.ArgumentParser(description="Run tests against a document.") parser.add_argument("--file", nargs="+", required=True, help="List of file paths to test.") @@ -793,21 +787,3 @@ async def main(): if __name__ == "__main__": asyncio.run(main()) - # delete_mems = await memory.dynamic_method_call(dynamic_memory_class, 'delete_memories', - # namespace=test_id) - # test_load_pipeline = await asyncio.gather( - # *(run_load_test_element(test_item,loader_settings, metadata, test_id) for test_item in test_set) - # ) - # - # test_eval_pipeline = await asyncio.gather( - # *(run_search_eval_element(test_item, test_id) for test_item in test_set) - # ) - # logging.info("Results of the eval pipeline %s", str(test_eval_pipeline)) - # await add_entity(session, TestOutput(id=test_id, user_id=user_id, test_results=str(test_eval_pipeline))) - # return test_eval_pipeline - -# # Gather and run all tests in parallel -# results = await asyncio.gather( -# *(run_testo(test, loader_settings, metadata) for test in test_params) -# ) -# return results