From 12e3c4d6b60d6dc4ff98cdf80ef4a0420b58f1ba Mon Sep 17 00:00:00 2001 From: Will-amSun Date: Mon, 16 Dec 2024 17:57:33 -0500 Subject: [PATCH] Finished overall checking, reviewing reworked implementations still needed. --- MISOReports/MISOReports.py | 26 +++++++++++++------------- MISOReports/parsers.py | 14 +++++++++++--- MISOReports/test_MISOReports.py | 2 +- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/MISOReports/MISOReports.py b/MISOReports/MISOReports.py index bcd071c..2ea2a52 100644 --- a/MISOReports/MISOReports.py +++ b/MISOReports/MISOReports.py @@ -1838,7 +1838,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2022, month=1, day=1), ), - "da_bc_HIST": Report( + "da_bc_HIST": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="da_bc_HIST", supported_extensions=["csv"], @@ -1851,7 +1851,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2022, month=1, day=1), ), - "da_ex_rg": Report( + "da_ex_rg": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="da_ex_rg", supported_extensions=["xlsx"], @@ -1864,7 +1864,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2024, month=9, day=1), ), - "da_ex": Report( + "da_ex": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="da_ex", supported_extensions=["xls"], @@ -1877,7 +1877,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2022, month=1, day=1), ), - "da_rpe": Report( + "da_rpe": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="da_rpe", supported_extensions=["xls"], @@ -1890,7 +1890,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2024, month=10, day=20), ), - "RT_LMPs": Report( + "RT_LMPs": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="RT_LMPs", supported_extensions=["zip"], @@ -1903,7 +1903,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2023, month=10, day=1), ), - "sr_gfm": Report( + "sr_gfm": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="sr_gfm", supported_extensions=["xlsx"], @@ -1916,7 +1916,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2024, month=9, day=1), ), - "dfal_HIST": Report( + "dfal_HIST": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="dfal_HIST", supported_extensions=["xls"], @@ -1929,7 +1929,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2022, month=12, day=31), ), - "historical_gen_fuel_mix": Report( + "historical_gen_fuel_mix": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="historical_gen_fuel_mix", supported_extensions=["xlsx"], @@ -1942,7 +1942,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2022, month=1, day=1), ), - "hwd_HIST": Report( + "hwd_HIST": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="hwd_HIST", supported_extensions=["csv"], @@ -1955,7 +1955,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2022, month=12, day=31), ), - "sr_hist_is": Report( + "sr_hist_is": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="sr_hist_is", supported_extensions=["csv"], @@ -1968,7 +1968,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2021, month=1, day=1), ), - "rfal_HIST": Report( + "rfal_HIST": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="rfal_HIST", supported_extensions=["xls"], @@ -1981,7 +1981,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2022, month=12, day=31), ), - "sr_lt": Report( + "sr_lt": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="sr_lt", supported_extensions=["xls"], @@ -2007,7 +2007,7 @@ def add_to_datetime( example_datetime=datetime.datetime(year=2022, month=1, day=1), ), - "mom": Report( + "mom": Report( # Checked 2024-12-16. url_builder=MISOMarketReportsURLBuilder( target="mom", supported_extensions=["xlsx"], diff --git a/MISOReports/parsers.py b/MISOReports/parsers.py index 3ae1185..3c8749c 100644 --- a/MISOReports/parsers.py +++ b/MISOReports/parsers.py @@ -2661,7 +2661,7 @@ def parse_da_bc_HIST( res: requests.Response, ) -> pd.DataFrame: text = res.text - csv_data = "\n".join(text.splitlines()[2:-3]) + csv_data = "\n".join(text.splitlines()[2:-2]) df = pd.read_csv( filepath_or_buffer=io.StringIO(csv_data), @@ -2706,7 +2706,8 @@ def parse_da_ex_rg( ).iloc[:-1] df.dropna(subset=['Region'], inplace=True) - + df = df.reset_index(drop=True) + last_value = None filled_column = [] @@ -2954,8 +2955,15 @@ def parse_hwd_HIST( filepath_or_buffer=io.StringIO(csv_data), ) + df.rename( + columns={ + "Market Day ": "Market Day", + }, + inplace=True, + ) + df[["Hour Ending"]] = df[["Hour Ending"]].astype("Int64") - df[["Market Day "]] = df[["Market Day "]].apply(pd.to_datetime, format="%m/%d/%Y") + df[["Market Day"]] = df[["Market Day"]].apply(pd.to_datetime, format="%m/%d/%Y") df[["MWh"]] = df[["MWh"]].astype("Float64") return df diff --git a/MISOReports/test_MISOReports.py b/MISOReports/test_MISOReports.py index a67d708..ffd4227 100644 --- a/MISOReports/test_MISOReports.py +++ b/MISOReports/test_MISOReports.py @@ -936,7 +936,7 @@ def test_MISOMarketReportsURLBuilder_build_url( { ("MWh",): pd.api.types.is_float_dtype, ("Hour Ending",): pd.api.types.is_integer_dtype, - ("Market Day ",): pd.api.types.is_datetime64_ns_dtype, + ("Market Day",): pd.api.types.is_datetime64_ns_dtype, } ), (