From 43a833c99837534d8e5ba1a5fdb0a2c4811fdd59 Mon Sep 17 00:00:00 2001 From: Brett Elliot Date: Thu, 28 Nov 2024 11:59:20 -0500 Subject: [PATCH 1/3] fix for bug in data.py for polygon data --- lumibot/entities/data.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lumibot/entities/data.py b/lumibot/entities/data.py index dbc15018e..1e3b8c539 100644 --- a/lumibot/entities/data.py +++ b/lumibot/entities/data.py @@ -480,6 +480,11 @@ def _get_bars_dict(self, dt, length=1, timestep=None, timeshift=0): # Get bars. end_row = self.get_iter_count(dt) - timeshift + if self.df.index[end_row] != dt: + # If dt is not in the dataframe, get_iter_count will return the last bar before dt. + # Since the data is not complete, we need to get the last bar, which is the end_row. + # And since the selection at the end is exclusive of end_row, we need to add 1 to end_row here. + end_row += 1 start_row = end_row - length if start_row < 0: From f05b327e2e5f18b17e27ea0f5ea02930a95a6392 Mon Sep 17 00:00:00 2001 From: Brett Elliot Date: Mon, 9 Dec 2024 08:49:56 -0500 Subject: [PATCH 2/3] Make test deterministic --- lumibot/entities/data.py | 5 ----- tests/test_get_historical_prices.py | 17 ++++++----------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/lumibot/entities/data.py b/lumibot/entities/data.py index 06873a5f0..b040e7e1b 100644 --- a/lumibot/entities/data.py +++ b/lumibot/entities/data.py @@ -480,11 +480,6 @@ def _get_bars_dict(self, dt, length=1, timestep=None, timeshift=0): # Get bars. end_row = self.get_iter_count(dt) - timeshift - if self.df.index[end_row] != dt: - # If dt is not in the dataframe, get_iter_count will return the last bar before dt. - # Since the data is not complete, we need to get the last bar, which is the end_row. - # And since the selection at the end is exclusive of end_row, we need to add 1 to end_row here. - end_row += 1 start_row = end_row - length if start_row < 0: diff --git a/tests/test_get_historical_prices.py b/tests/test_get_historical_prices.py index 4a19037c0..528a78fdc 100644 --- a/tests/test_get_historical_prices.py +++ b/tests/test_get_historical_prices.py @@ -16,9 +16,7 @@ from lumibot.tools import get_trading_days # Global parameters -# API Key for testing Polygon.io -from lumibot.credentials import POLYGON_API_KEY -from lumibot.credentials import TRADIER_CONFIG, ALPACA_CONFIG +from lumibot.credentials import TRADIER_CONFIG, ALPACA_CONFIG, POLYGON_CONFIG logger = logging.getLogger(__name__) @@ -134,13 +132,13 @@ def test_pandas_backtesting_data_source_get_historical_prices_daily_bars(self, p self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start(bars, backtesting_start=backtesting_start) self.check_dividends_and_adjusted_returns(bars) - @pytest.mark.skip(reason="This test exposes a possible bug in data.py that we have not investigated yet.") - @pytest.mark.skipif(POLYGON_API_KEY == '', reason="This test requires a Polygon.io API key") + @pytest.mark.skipif(not POLYGON_CONFIG["API_KEY"], reason="This test requires a Polygon.io API key") + @pytest.mark.skipif(not POLYGON_CONFIG["IS_PAID_SUBSCRIPTION"], reason="This test requires a paid Polygon.io API key") def test_polygon_backtesting_data_source_get_historical_prices_daily_bars(self): - backtesting_end = datetime.now() - timedelta(days=1) - backtesting_start = backtesting_end - timedelta(days=self.length * 2 + 5) + backtesting_start = datetime(2019, 3, 25) + backtesting_end = datetime(2019, 4, 25) data_source = PolygonDataBacktesting( - backtesting_start, backtesting_end, api_key=POLYGON_API_KEY + backtesting_start, backtesting_end, api_key=POLYGON_CONFIG["API_KEY"] ) bars = data_source.get_historical_prices(asset=self.asset, length=self.length, timestep=self.timestep) check_bars(bars=bars, length=self.length) @@ -164,7 +162,6 @@ def test_yahoo_backtesting_data_source_get_historical_prices_daily_bars(self, pa self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start(bars, backtesting_start=backtesting_start) -# @pytest.mark.skip() class TestDatasourceGetHistoricalPricesDailyData: """These tests check the daily Bars returned from get_historical_prices for live data sources.""" @@ -202,7 +199,6 @@ def check_date_of_last_bar_is_correct_for_live_data_sources(self, bars): # if it's not a trading day, the last bar the bar should from the last trading day assert bars.df.index[-1].date() == self.trading_days.index[-1].date() - # @pytest.mark.skip() @pytest.mark.skipif(not ALPACA_CONFIG['API_KEY'], reason="This test requires an alpaca API key") @pytest.mark.skipif( ALPACA_CONFIG['API_KEY'] == '', @@ -225,7 +221,6 @@ def test_alpaca_data_source_get_historical_prices_daily_bars(self): check_bars(bars=bars, length=1, check_timezone=False) self.check_date_of_last_bar_is_correct_for_live_data_sources(bars) - # @pytest.mark.skip() @pytest.mark.skipif(not TRADIER_CONFIG['ACCESS_TOKEN'], reason="No Tradier credentials provided.") def test_tradier_data_source_get_historical_prices_daily_bars(self): data_source = TradierData( From 75253925e3f389e31b46581606d99b7853b42d3e Mon Sep 17 00:00:00 2001 From: Brett Elliot Date: Mon, 9 Dec 2024 09:16:51 -0500 Subject: [PATCH 3/3] Added deterministic tests for thanksgiving dates. --- lumibot/entities/data.py | 5 +++++ tests/test_get_historical_prices.py | 34 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/lumibot/entities/data.py b/lumibot/entities/data.py index b040e7e1b..06873a5f0 100644 --- a/lumibot/entities/data.py +++ b/lumibot/entities/data.py @@ -480,6 +480,11 @@ def _get_bars_dict(self, dt, length=1, timestep=None, timeshift=0): # Get bars. end_row = self.get_iter_count(dt) - timeshift + if self.df.index[end_row] != dt: + # If dt is not in the dataframe, get_iter_count will return the last bar before dt. + # Since the data is not complete, we need to get the last bar, which is the end_row. + # And since the selection at the end is exclusive of end_row, we need to add 1 to end_row here. + end_row += 1 start_row = end_row - length if start_row < 0: diff --git a/tests/test_get_historical_prices.py b/tests/test_get_historical_prices.py index 528a78fdc..00703251a 100644 --- a/tests/test_get_historical_prices.py +++ b/tests/test_get_historical_prices.py @@ -132,6 +132,18 @@ def test_pandas_backtesting_data_source_get_historical_prices_daily_bars(self, p self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start(bars, backtesting_start=backtesting_start) self.check_dividends_and_adjusted_returns(bars) + # First trading day after Thanksgiving test + backtesting_start = datetime(2019, 11, 2) + backtesting_end = datetime(2019, 12, 2) + data_source = PandasData( + datetime_start=backtesting_start, + datetime_end=backtesting_end, + pandas_data=pandas_data_fixture + ) + bars = data_source.get_historical_prices(asset=self.asset, length=self.length, timestep=self.timestep) + check_bars(bars=bars, length=self.length) + self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start(bars, backtesting_start=backtesting_start) + @pytest.mark.skipif(not POLYGON_CONFIG["API_KEY"], reason="This test requires a Polygon.io API key") @pytest.mark.skipif(not POLYGON_CONFIG["IS_PAID_SUBSCRIPTION"], reason="This test requires a paid Polygon.io API key") def test_polygon_backtesting_data_source_get_historical_prices_daily_bars(self): @@ -144,6 +156,16 @@ def test_polygon_backtesting_data_source_get_historical_prices_daily_bars(self): check_bars(bars=bars, length=self.length) self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start(bars, backtesting_start=backtesting_start) + # First trading day after Thanksgiving test + backtesting_start = datetime(2019, 3, 25) + backtesting_end = datetime(2019, 4, 25) + data_source = PolygonDataBacktesting( + backtesting_start, backtesting_end, api_key=POLYGON_CONFIG["API_KEY"] + ) + bars = data_source.get_historical_prices(asset=self.asset, length=self.length, timestep=self.timestep) + check_bars(bars=bars, length=self.length) + self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start(bars, backtesting_start=backtesting_start) + def test_yahoo_backtesting_data_source_get_historical_prices_daily_bars(self, pandas_data_fixture): """ This tests that the yahoo data_source calculates adjusted returns for bars and that they @@ -161,6 +183,18 @@ def test_yahoo_backtesting_data_source_get_historical_prices_daily_bars(self, pa self.check_dividends_and_adjusted_returns(bars) self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start(bars, backtesting_start=backtesting_start) + # First trading day after Thanksgiving test + backtesting_start = datetime(2019, 3, 25) + backtesting_end = datetime(2019, 4, 25) + data_source = YahooDataBacktesting( + datetime_start=backtesting_start, + datetime_end=backtesting_end, + pandas_data=pandas_data_fixture + ) + bars = data_source.get_historical_prices(asset=self.asset, length=self.length, timestep=self.timestep) + check_bars(bars=bars, length=self.length) + self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start(bars, backtesting_start=backtesting_start) + class TestDatasourceGetHistoricalPricesDailyData: """These tests check the daily Bars returned from get_historical_prices for live data sources."""