Features/gjp miner

README.md

@@ -190,6 +190,7 @@ pm2 start pm2_configs/validator.config.js
 ```
 Keep in mind that you will need to change the default parameters for either the [miner](./scripts/run_miner.sh) or the [validator](./scripts/run_validator.sh). 
 
+Miners now have the opportunity to interact with the global job pool (GJP) locally. By creating a read-only node via `start_read_node.sh`, miners sync with the GJP and create snapshots on their local machine. We have provided a function in `folding/miners/folding_miner.py` that returns jobs based on their priority in the GJP. With this information, you can experiment with customizing your job queue. 
 ## How does the Subnet Work?
 
 In this subnet, validators create protein folding challenges for miners, who in turn run simulations using OpenMM to obtain stable protein configurations. At a high level, each role can be broken down into parts: 

db/rsnapshots/7-15997-1736885884838/meta.json

@@ -0,0 +1 @@
+{"Version":1,"ID":"7-15997-1736885884838","Index":15997,"Term":7,"Peers":null,"Configuration":{"Servers":[{"Suffrage":0,"ID":"1","Address":"174.138.3.61:4002"},{"Suffrage":1,"ID":"gjp-miner","Address":"184.105.4.4:4002"}]},"ConfigurationIndex":15359,"Size":5967872}

folding/miners/folding_miner.py

@@ -9,6 +9,8 @@
 from typing import Dict, List, Tuple
 import copy
 import traceback
+import asyncio 
+import sqlite3
 
 import bittensor as bt
 import openmm as mm
@@ -18,7 +20,6 @@
 from folding.base.miner import BaseMinerNeuron
 from folding.base.simulation import OpenMMSimulation
 from folding.protocol import JobSubmissionSynapse
-from folding.utils.logging import log_event
 from folding.utils.reporters import ExitFileReporter, LastTwoCheckpointsReporter
 from folding.utils.ops import (
     check_if_directory_exists,
@@ -120,6 +121,57 @@ def check_synapse(
     return synapse
 
 
+def check_sqlite_table(db_path: str, max_workers: int) -> Dict:
+    """
+    Fetches a limited number of job records from an SQLite database, ordering them by priority.
+
+    This function connects to an SQLite database at the given path and retrieves job details
+    based on the specified maximum number of worker processes. It returns a dictionary where each
+    key is the unique job 'id' and the value is another dictionary containing selected details
+    of the job such as job ID, pdb ID, creation date, priority, organic flag, s3 links, and system configuration.
+
+    Parameters:
+        db_path (str): The file path to the SQLite database.
+        max_workers (int): The maximum number of job records to fetch, which are sorted by priority in descending order.
+
+    Returns:
+        Dict: A dictionary where each key is a job 'id' and the value is another dictionary with job details.
+
+    Raises:
+        sqlite3.Error: If there is an error fetching data from the database, it logs the error message.
+
+    """
+    logger.info("Checking sqlite table for jobs")
+    # Define the columns to be selected
+    columns_to_select = "id, job_id, pdb_id, created_at, priority, is_organic, s3_links, system_config"
+    try:
+        with sqlite3.connect(db_path) as conn:
+            cursor = conn.cursor()
+
+            query = f"SELECT {columns_to_select} from jobs ORDER BY priority DESC LIMIT ?"
+            cursor.execute(query, (max_workers,))
+
+            selected_columns = [description[0] for description in cursor.description]
+
+            # jobs = tuple containing all selected jobs
+            jobs = cursor.fetchall()
+            if not jobs:
+                logger.info("No jobs found in local snapshot of GJP")
+                return {}
+
+            jobs_dict = {}
+            for job in jobs:
+                job_dict = {selected_columns[i]: job[i] for i in range(len(job))}
+                job_id = job_dict.pop('id')  
+                jobs_dict[job_id] = job_dict
+            cursor.close()
+
+            return jobs_dict
+
+    except sqlite3.Error as e:
+        logger.info(f"Error fetching sqlite data: {e}")
+
+
 class FoldingMiner(BaseMinerNeuron):
     def __init__(self, config=None, base_data_path: str = None):
         super().__init__(config=config)
@@ -146,6 +198,7 @@ def __init__(self, config=None, base_data_path: str = None):
 
         self.mock = None
         self.generate_random_seed = lambda: random.randint(0, 1000)
+        self.db_path = "/db/db.sqlite"
 
         # hardcorded for now -- TODO: make this more flexible
         self.STATES = ["nvt", "npt", "md_0_1"]
@@ -362,6 +415,7 @@ def forward(self, synapse: JobSubmissionSynapse) -> JobSubmissionSynapse:
             elif len(synapse.md_inputs) == 0:  # The vali sends nothing to the miner
                 return check_synapse(self=self, synapse=synapse, event=event)
 
+
     def submit_simulation(
         self,
         synapse: JobSubmissionSynapse,

install.sh

@@ -41,4 +41,5 @@ poetry install
 sudo apt-get update
 sudo apt-get install build-essential cmake libfftw3-dev vim npm -y
 sudo npm install -g pm2 -y
+chmod +x install_rqlite.sh
 ./install_rqlite.sh

tests/test_miner_functions.py

@@ -0,0 +1,43 @@
+import pytest 
+from folding.miners.folding_miner import check_sqlite_table 
+import unittest
+from unittest.mock import patch, MagicMock
+import sqlite3
+
+class TestCheckSqliteTable(unittest.TestCase):
+    def setUp(self):
+        self.sample_data = [
+                (1, 'job1', 'pdb1', '2025-01-01 00:00:00', 10, True, '{"pdb": "link_to_pdb"}', '{"ff": "file.xml"}'),
+                (2, 'job2', 'pdb2', '2025-01-02 00:00:00', 5, False, '{"pdb": "link_to_pdb2"}', '{"ff": "file2.xml"}')
+            ]
+        self.columns = ['id', 'job_id', 'pdb_id', 'created_at', 'priority', 'is_organic', 's3_links', 'system_config']
+
+    @patch('folding.utils.logger.logger')
+    @patch('sqlite3.connect')
+    def test_check_sqlite_table_success(self, mock_connect, mock_logger):
+        mock_cursor = MagicMock()
+        mock_cursor.fetchall.return_value = self.sample_data
+        mock_cursor.description = [(column,) for column in self.columns]
+        mock_connect.return_value.__enter__.return_value.cursor.return_value = mock_cursor
+
+        result = check_sqlite_table(db_path='test_path.db', max_workers=2)
+        print(len(result))
+        # check if result is correctly formatted 
+        self.assertIsInstance(result, dict)
+        self.assertEqual(len(result), 2)
+        self.assertTrue(all(isinstance(result[key], dict) for key in result))
+
+        # check if all expected keys are in the results
+        expected_keys = ['job_id', 'pdb_id', 'created_at', 'priority', 'is_organic', 's3_links', 'system_config']
+        for job_details in result.values():
+            self.assertTrue(all(key in job_details for key in expected_keys))
+
+    @patch('folding.utils.logger.logger')
+    @patch('sqlite3.connect')
+    def test_check_sqlite_table_error(self, mock_connect, mock_logger):
+        mock_connect.side_effect = sqlite3.Error("Fake SQL Error")
+
+        result = check_sqlite_table(db_path='test_path.db', max_workers=2)
+
+        # check if result is None
+        self.assertIsNone(result)