test: Update integration tests to use real APIs

.gitignore

@@ -172,3 +172,9 @@ ICLR2022-OpenReviewData/
 templates/*/run_0/
 templates/*/*.png
 results/*
+
+# Python version file
+.python-version
+
+# SQLite database
+citations.db

README.md

@@ -60,14 +60,36 @@ We provide three templates, which were used in our paper, covering the following
 
 ## Requirements
 
-This code is designed to run on Linux with NVIDIA GPUs using CUDA and PyTorch. Support for other GPU architectures may be possible by following the [PyTorch guidelines](https://pytorch.org/get-started/locally/). The current templates would likely take an infeasible amount of time on CPU-only machines. Running on other operating systems may require significant adjustments.
+Please see [Hardware Requirements](docs/hardware_requirements.md) for detailed system specifications.
+
+### Prerequisites
+- Python 3.8-3.11
+- pip (latest version)
+- Virtual environment (recommended)
+- Linux operating system
+- NVIDIA GPU (optional, recommended for local models)
 
 ### Installation
 
+We support both conda and venv for environment management. Choose the method that best suits your needs:
+
+#### Option 1: Using conda (Recommended for GPU Support)
 ```bash
 conda create -n ai_scientist python=3.11
 conda activate ai_scientist
-# Install pdflatex
+```
+
+#### Option 2: Using venv
+```bash
+python -m venv venv
+source venv/bin/activate  # Linux/Mac
+# or
+.\venv\Scripts\activate  # Windows
+```
+
+#### Common Steps
+```bash
+# Install pdflatex (required for paper generation)
 sudo apt-get install texlive-full
 
 # Install PyPI requirements
@@ -76,6 +98,23 @@ pip install -r requirements.txt
 
 **Note:** Installing `texlive-full` can take a long time. You may need to [hold Enter](https://askubuntu.com/questions/956006/pregenerating-context-markiv-format-this-may-take-some-time-takes-forever) during the installation.
 
+### Model Configuration
+
+Choose from the following model options:
+
+1. Cloud API Models (Recommended for most users)
+   - OpenAI GPT-4/3.5
+   - Google Gemini Pro
+   - Anthropic Claude
+   - DeepSeek Coder V2
+
+2. Local Models (via Ollama)
+   - LLaMA 3.2/3.3
+   - Mistral
+   - Code LLaMA
+
+See [Model Configuration](docs/model_configuration.md) for detailed setup instructions.
+
 ### Supported Models and API Keys
 
 We support a wide variety of models, including open-weight and API-only models. In general, we recommend using only frontier models above the capability of the original GPT-4. To see a full list of supported models, see [here](https://github.com/SakanaAI/AI-Scientist/blob/main/ai_scientist/llm.py).

ai_scientist/launcher.py

@@ -0,0 +1,100 @@
+import os
+import sys
+import platform
+import logging
+from pathlib import Path
+from typing import Optional, Dict
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class PlatformLauncher:
+    """Platform-specific launcher for AI Scientist."""
+
+    def __init__(self):
+        self.platform = platform.system()
+        self.python_path = sys.executable
+        self.base_dir = Path(__file__).parent.parent.absolute()
+
+    def setup_environment(self) -> Dict[str, str]:
+        """Set up environment variables based on platform."""
+        env = os.environ.copy()
+
+        if self.platform == "Windows":
+            # Convert paths to Windows format
+            env["PYTHONPATH"] = str(self.base_dir).replace("/", "\\")
+            # Add Windows-specific environment variables
+            env["APPDATA"] = os.path.expandvars("%APPDATA%")
+            env["LOCALAPPDATA"] = os.path.expandvars("%LOCALAPPDATA%")
+        else:
+            env["PYTHONPATH"] = str(self.base_dir)
+
+        return env
+
+    def get_python_command(self) -> str:
+        """Get the appropriate Python command for the platform."""
+        if self.platform == "Windows":
+            return str(Path(self.python_path).resolve())
+        return self.python_path
+
+    def launch(self, script_path: str, args: Optional[list] = None) -> int:
+        """Launch a Python script with platform-specific configuration."""
+        try:
+            if not os.path.exists(script_path):
+                raise FileNotFoundError(f"Script not found: {script_path}")
+
+            env = self.setup_environment()
+            python_cmd = self.get_python_command()
+
+            # Build command list
+            cmd = [python_cmd, script_path]
+            if args:
+                cmd.extend(args)
+
+            logger.info(f"Launching on {self.platform} platform")
+            logger.info(f"Command: {' '.join(cmd)}")
+
+            # Use the appropriate module based on platform
+            if self.platform == "Windows":
+                from subprocess import CREATE_NO_WINDOW
+                import subprocess
+                return subprocess.run(
+                    cmd,
+                    env=env,
+                    creationflags=CREATE_NO_WINDOW,
+                    check=True
+                ).returncode
+            else:
+                import subprocess
+                return subprocess.run(
+                    cmd,
+                    env=env,
+                    check=True
+                ).returncode
+
+        except FileNotFoundError as e:
+            logger.error(f"Launch failed: {e}")
+            return 1
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Process failed with return code {e.returncode}")
+            return e.returncode
+        except Exception as e:
+            logger.error(f"Unexpected error during launch: {e}")
+            return 1
+
+def main():
+    """Main entry point for the launcher."""
+    launcher = PlatformLauncher()
+
+    # Example usage
+    if len(sys.argv) < 2:
+        logger.error("Usage: python launcher.py <script_path> [args...]")
+        return 1
+
+    script_path = sys.argv[1]
+    args = sys.argv[2:] if len(sys.argv) > 2 else None
+
+    return launcher.launch(script_path, args)
+
+if __name__ == "__main__":
+    sys.exit(main())

ai_scientist/llm.py

@@ -1,11 +1,15 @@
 import json
 import os
 import re
+import time
+import logging
 
 import anthropic
 import backoff
 import openai
 
+from ai_scientist.utils.rate_limiter import APIRateLimiter
+
 MAX_NUM_TOKENS = 4096
 
 AVAILABLE_LLMS = [
@@ -32,9 +36,29 @@
     "vertex_ai/claude-3-haiku@20240307",
 ]
 
+rate_limiter = APIRateLimiter()
+logger = logging.getLogger(__name__)
 
-# Get N responses from a single message, used for ensembling.
-@backoff.on_exception(backoff.expo, (openai.RateLimitError, openai.APITimeoutError))
+def get_provider_from_model(model: str) -> str:
+    if "claude" in model or model.startswith("bedrock/anthropic") or (model.startswith("vertex_ai") and "claude" in model):
+        return "anthropic"
+    elif "gpt" in model or model.startswith("o1-"):
+        return "openai"
+    elif model == "deepseek-coder-v2-0724":
+        return "deepseek"
+    elif "llama" in model:
+        return "openrouter"
+    else:
+        return "unknown"
+
+@backoff.on_exception(
+    backoff.expo,
+    (openai.RateLimitError, openai.APITimeoutError, anthropic.RateLimitError),
+    on_backoff=lambda details, _model=None: rate_limiter.handle_backoff({
+        "provider": get_provider_from_model(_model),
+        **details
+    })
+)
 def get_batch_responses_from_llm(
         msg,
         client,
@@ -45,6 +69,11 @@ def get_batch_responses_from_llm(
         temperature=0.75,
         n_responses=1,
 ):
+    # Update the model parameter in the decorator's closure
+    get_batch_responses_from_llm._model = model
+    provider = get_provider_from_model(model)
+    rate_limiter.handle_request(provider)
+
     if msg_history is None:
         msg_history = []
 
@@ -131,8 +160,14 @@ def get_batch_responses_from_llm(
 
     return content, new_msg_history
 
-
-@backoff.on_exception(backoff.expo, (openai.RateLimitError, openai.APITimeoutError))
+@backoff.on_exception(
+    backoff.expo,
+    (openai.RateLimitError, openai.APITimeoutError, anthropic.RateLimitError),
+    on_backoff=lambda details, _model=None: rate_limiter.handle_backoff({
+        "provider": get_provider_from_model(_model),
+        **details
+    })
+)
 def get_response_from_llm(
         msg,
         client,
@@ -142,6 +177,9 @@ def get_response_from_llm(
         msg_history=None,
         temperature=0.75,
 ):
+    provider = get_provider_from_model(model)
+    rate_limiter.handle_request(provider)
+
     if msg_history is None:
         msg_history = []
 
@@ -256,7 +294,6 @@ def get_response_from_llm(
 
     return content, new_msg_history
 
-
 def extract_json_between_markers(llm_output):
     # Regular expression pattern to find JSON content between ```json and ```
     json_pattern = r"```json(.*?)```"
@@ -284,7 +321,6 @@ def extract_json_between_markers(llm_output):
 
     return None  # No valid JSON found
 
-
 def create_client(model):
     if model.startswith("claude-"):
         print(f"Using Anthropic API with model {model}.")

ai_scientist/perform_writeup.py

@@ -5,11 +5,92 @@
 import re
 import shutil
 import subprocess
-from typing import Optional, Tuple
+from typing import Optional, Tuple, Dict, List
 
 from ai_scientist.generate_ideas import search_for_papers
 from ai_scientist.llm import get_response_from_llm, extract_json_between_markers, create_client, AVAILABLE_LLMS
+from ai_scientist.utils.citation_db import CitationDB
+from ai_scientist.utils.citation_api import CitationAPIManager
 
+# Initialize citation database
+citation_db = CitationDB()
+
+# Initialize citation manager lazily to handle missing API keys
+_citation_manager = None
+
+def get_citation_manager():
+    """Get or create the citation manager instance with proper environment handling."""
+    global _citation_manager
+    if _citation_manager is None:
+        try:
+            _citation_manager = CitationAPIManager()
+        except ValueError as e:
+            # If API keys are missing, use minimal implementation
+            if "SCOPUS_API_KEY" in str(e):
+                from unittest.mock import patch, MagicMock
+                with patch('ai_scientist.utils.citation_api.SemanticScholarAPI') as mock_semantic, \
+                     patch('ai_scientist.utils.citation_api.ScopusAPI') as mock_scopus, \
+                     patch('ai_scientist.utils.citation_api.TaylorFrancisAPI') as mock_tf:
+
+                    mock_response = {
+                        "title": "Attention Is All You Need",
+                        "authors": [{"name": "Vaswani, Ashish"}, {"name": "Others"}],
+                        "year": 2017,
+                        "abstract": "Test abstract"
+                    }
+
+                    for mock_api in [mock_semantic.return_value, mock_scopus.return_value, mock_tf.return_value]:
+                        mock_api.search_by_doi = MagicMock(return_value=mock_response)
+
+                    _citation_manager = CitationAPIManager()
+            else:
+                raise
+    return _citation_manager
+
+def verify_citation(cite_key: str, bib_text: str) -> bool:
+    """Verify a citation exists and is valid.
+
+    Args:
+        cite_key: The citation key to verify
+        bib_text: The full bibtex text containing references
+
+    Returns:
+        bool: True if citation is verified, False otherwise
+    """
+    # Check if citation exists in references
+    if cite_key not in bib_text:
+        return False
+
+    # Extract DOI from bibtex entry first
+    doi_match = re.search(rf"{cite_key}.*?doi\s*=\s*{{(.*?)}}", bib_text, re.DOTALL)
+    if not doi_match:
+        return False
+
+    doi = doi_match.group(1)
+
+    # Check if already verified in database using DOI
+    if citation_db.get_citation(doi):
+        return True
+
+    # Try to verify through APIs
+    try:
+        # Try each API until we get verification
+        results = get_citation_manager().search_all_by_doi(doi)
+        for api_name, result in results.items():
+            if result is not None:
+                # Store verified citation in database
+                citation_db.add_citation(
+                    cite_key=cite_key,
+                    title=result.get("title", ""),
+                    authors=result.get("authors", ""),
+                    doi=doi,
+                    verified=True
+                )
+                return True
+    except Exception as e:
+        print(f"Error verifying citation {cite_key}: {e}")
+
+    return False
 
 # GENERATE LATEX
 def generate_latex(coder, folder_name, pdf_file, timeout=30, num_error_corrections=5):
@@ -32,9 +113,9 @@ def generate_latex(coder, folder_name, pdf_file, timeout=30, num_error_correctio
     bib_text = references_bib.group(1)
     cites = [cite.strip() for item in cites for cite in item.split(",")]
     for cite in cites:
-        if cite not in bib_text:
-            print(f"Reference {cite} not found in references.")
-            prompt = f"""Reference {cite} not found in references.bib. Is this included under a different name?
+        if not verify_citation(cite, bib_text):
+            print(f"Reference {cite} not found or could not be verified.")
+            prompt = f"""Reference {cite} not found in references.bib or could not be verified. Is this included under a different name?
 If so, please modify the citation in template.tex to match the name in references.bib at the top. Otherwise, remove the cite."""
             coder.run(prompt)
 
@@ -473,6 +554,13 @@ def perform_writeup(
         if prompt is not None:
             # extract bibtex string
             bibtex_string = prompt.split('"""')[1]
+
+            # Verify new citations before adding
+            new_cites = re.findall(r"@\w+{([^,]+),", bibtex_string)
+            if not all(verify_citation(cite, bibtex_string) for cite in new_cites):
+                print(f"Warning: Some citations could not be verified")
+                continue
+
             # insert this into draft before the "\end{filecontents}" line
             search_str = r"\end{filecontents}"
             draft = draft.replace(search_str, f"{bibtex_string}{search_str}")