diff --git a/docs/pages/benchmark.md b/docs/pages/benchmark.md new file mode 100644 index 000000000..6690a7e7b --- /dev/null +++ b/docs/pages/benchmark.md @@ -0,0 +1,6 @@ +# Benchmark your model as a social agent in Sotopia + +``` +python sotopia/benchmark/cli.py --model= +``` +Currently this script would run over 100 simulations on the Sotopia Hard tasks. And the partner model is fixed to be `together_ai/meta-llama/Llama-2-70b-chat-hf` \ No newline at end of file diff --git a/sotopia_conf/server.py b/sotopia_conf/server.py index 2f502a0ca..d8d9fe263 100644 --- a/sotopia_conf/server.py +++ b/sotopia_conf/server.py @@ -22,7 +22,7 @@ def main(_: Any) -> None: FLAGS.gin_bindings, ) - asyncio.run(run_async_server()) # type: ignore[call-arg] + asyncio.run(run_async_server()) if __name__ == "__main__":