Finished overall checking, reviewing reworked implementations still n…

…eeded.
BrianWeiHaoMa · Dec 16, 2024 · 12e3c4d · 12e3c4d
1 parent 1119457
commit 12e3c4d
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 17 deletions.
diff --git a/MISOReports/MISOReports.py b/MISOReports/MISOReports.py
@@ -1838,7 +1838,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2022, month=1, day=1),
         ),
 
-        "da_bc_HIST": Report(
+        "da_bc_HIST": Report( # Checked 2024-12-16.
                 url_builder=MISOMarketReportsURLBuilder(
                 target="da_bc_HIST",
                 supported_extensions=["csv"],
@@ -1851,7 +1851,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2022, month=1, day=1),
         ),
 
-        "da_ex_rg": Report(
+        "da_ex_rg": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="da_ex_rg",
                 supported_extensions=["xlsx"],
@@ -1864,7 +1864,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2024, month=9, day=1),
         ),
 
-        "da_ex": Report(
+        "da_ex": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="da_ex",
                 supported_extensions=["xls"],
@@ -1877,7 +1877,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2022, month=1, day=1),
         ),
 
-        "da_rpe": Report(
+        "da_rpe": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="da_rpe",
                 supported_extensions=["xls"],
@@ -1890,7 +1890,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2024, month=10, day=20),
         ),
 
-        "RT_LMPs": Report(
+        "RT_LMPs": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="RT_LMPs",
                 supported_extensions=["zip"],
@@ -1903,7 +1903,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2023, month=10, day=1),
         ),
 
-        "sr_gfm": Report(
+        "sr_gfm": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="sr_gfm",
                 supported_extensions=["xlsx"],
@@ -1916,7 +1916,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2024, month=9, day=1),
         ),
 
-        "dfal_HIST": Report(
+        "dfal_HIST": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="dfal_HIST",
                 supported_extensions=["xls"],
@@ -1929,7 +1929,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2022, month=12, day=31),
         ),
 
-        "historical_gen_fuel_mix": Report(
+        "historical_gen_fuel_mix": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="historical_gen_fuel_mix",
                 supported_extensions=["xlsx"],
@@ -1942,7 +1942,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2022, month=1, day=1),
         ),
 
-        "hwd_HIST": Report(
+        "hwd_HIST": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="hwd_HIST",
                 supported_extensions=["csv"],
@@ -1955,7 +1955,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2022, month=12, day=31),
         ),
 
-        "sr_hist_is": Report(
+        "sr_hist_is": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="sr_hist_is",
                 supported_extensions=["csv"],
@@ -1968,7 +1968,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2021, month=1, day=1),
         ),
 
-        "rfal_HIST": Report(
+        "rfal_HIST": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="rfal_HIST",
                 supported_extensions=["xls"],
@@ -1981,7 +1981,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2022, month=12, day=31),
         ),
 
-        "sr_lt": Report(
+        "sr_lt": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="sr_lt",
                 supported_extensions=["xls"],
@@ -2007,7 +2007,7 @@ def add_to_datetime(
             example_datetime=datetime.datetime(year=2022, month=1, day=1),
         ),
 
-        "mom": Report(
+        "mom": Report( # Checked 2024-12-16.
             url_builder=MISOMarketReportsURLBuilder(
                 target="mom",
                 supported_extensions=["xlsx"],

diff --git a/MISOReports/parsers.py b/MISOReports/parsers.py
@@ -2661,7 +2661,7 @@ def parse_da_bc_HIST(
         res: requests.Response,
 ) -> pd.DataFrame:
     text = res.text
-    csv_data = "\n".join(text.splitlines()[2:-3])
+    csv_data = "\n".join(text.splitlines()[2:-2])
 
     df = pd.read_csv(
         filepath_or_buffer=io.StringIO(csv_data),
@@ -2706,7 +2706,8 @@ def parse_da_ex_rg(
     ).iloc[:-1]
 
     df.dropna(subset=['Region'], inplace=True)
-
+    df = df.reset_index(drop=True)
+
     last_value = None
     filled_column = []
 
@@ -2954,8 +2955,15 @@ def parse_hwd_HIST(
         filepath_or_buffer=io.StringIO(csv_data),
     )
 
+    df.rename(
+        columns={
+            "Market Day	": "Market Day",
+        }, 
+        inplace=True,
+    )
+
     df[["Hour Ending"]] = df[["Hour Ending"]].astype("Int64")
-    df[["Market Day	"]] = df[["Market Day	"]].apply(pd.to_datetime, format="%m/%d/%Y")
+    df[["Market Day"]] = df[["Market Day"]].apply(pd.to_datetime, format="%m/%d/%Y")
     df[["MWh"]] = df[["MWh"]].astype("Float64")
 
     return df

diff --git a/MISOReports/test_MISOReports.py b/MISOReports/test_MISOReports.py
@@ -936,7 +936,7 @@ def test_MISOMarketReportsURLBuilder_build_url(
         {
             ("MWh",): pd.api.types.is_float_dtype,
             ("Hour Ending",): pd.api.types.is_integer_dtype,
-            ("Market Day	",): pd.api.types.is_datetime64_ns_dtype,
+            ("Market Day",): pd.api.types.is_datetime64_ns_dtype,
         }
     ),
     (