Merge pull request #364 from macrocosm-os/refactor_hf_code

Change date from 27th. Reduce limit from 400 to 200 to make upload more brief( to be changed on 27th)

common/constants.py

@@ -37,4 +37,4 @@
 NO_TWITTER_URLS_DATE = datetime.datetime(2024, 12, 28, tzinfo=datetime.timezone.utc)  # December 28, 2024 UTC
 
 # HF reward activation date.
-HF_REWARD_DATE = datetime.datetime(2025, 1, 27, tzinfo=datetime.timezone.utc)  # January 27, 2025 UTC
+HF_REWARD_DATE = datetime.datetime(2025, 1, 25, hour=16, tzinfo=datetime.timezone.utc)  # January 25, 16:00 2025 UTC

huggingface_utils/huggingface_uploader.py

@@ -143,7 +143,7 @@ def get_data_for_huggingface_upload(self, source, last_upload):
                 FROM DataEntity
                 WHERE source = ?
                 ORDER BY datetime ASC
-                LIMIT 400000000
+                LIMIT 200000000
             """
             params = [source]
         else:
@@ -252,8 +252,8 @@ def upload_sql_to_huggingface(self) -> List[HuggingFaceMetadata]:
                         continue
 
                     bt.logging.info(f"Current total rows: {total_rows}")
-                    if total_rows >= 400_000_000:
-                        bt.logging.info(f"Reached 400 million rows limit for source {source}. Stopping upload.")
+                    if total_rows >= 200_000_000: # TODO
+                        bt.logging.info(f"Reached 200 million rows limit for source {source}. Stopping upload.")
                         break
 
                     last_upload = df['datetime'].max()