Fixes for traider.get_historical_prices, and tests for several data sources.

lumibot/data_sources/data_source.py

@@ -70,7 +70,9 @@ def get_historical_prices(
         self, asset, length, timestep="", timeshift=None, quote=None, exchange=None, include_after_hours=True
     ) -> Bars:
         """
-        Get bars for a given asset
+        Get bars for a given asset, going back in time from now, getting length number of bars by timestep.
+        For example, with a length of 10 and a timestep of "1day", and now timeshift, this
+        would return the last 10 daily bars.
 
         Parameters
         ----------

lumibot/data_sources/tradier_data.py

@@ -1,6 +1,6 @@
 import logging
 from collections import defaultdict
-from datetime import datetime, date
+from datetime import datetime, date, timedelta
 
 import pandas as pd
 import pytz
@@ -204,8 +204,8 @@ def get_historical_prices(
 
         if timestep == 'day' and timeshift is None:
             # What we really want is the last n bars, not the bars from the last n days.
-            # get twice as many days as we need to ensure we get enough bars
-            tcal_start_date = end_date - (td * length * 2)
+            # get twice as many days as we need to ensure we get enough bars, then add 3 days for long weekends
+            tcal_start_date = end_date - (td * length * 2 + timedelta(days=3))
             trading_days = get_trading_days(market='NYSE', start_date=tcal_start_date, end_date=end_date)
             # Filer out trading days when the market_open is after the end_date
             trading_days = trading_days[trading_days['market_open'] < end_date]
@@ -242,7 +242,7 @@ def get_historical_prices(
 
         # if type of index is date, convert it to timestamp with timezone info of "America/New_York"
         if isinstance(df.index[0], date):
-            df.index = pd.to_datetime(df.index, utc=True).tz_convert("America/New_York")
+            df.index = pd.to_datetime(df.index).tz_localize("America/New_York")
 
         # Convert the dataframe to a Bars object
         bars = Bars(df, self.SOURCE, asset, raw=df, quote=quote)

lumibot/data_sources/yahoo_data.py

@@ -85,7 +85,7 @@ def _pull_source_symbol_bars(
             data = self._append_data(asset, data)
 
         if timestep == "day":
-            # Get the last minute of self._datetime to get the current bar
+            # Get the previous days bar
             dt = self._datetime.replace(hour=23, minute=59, second=59, microsecond=999999)
             end = dt - timedelta(days=1)
         else:

lumibot/strategies/strategy_executor.py

@@ -867,6 +867,8 @@ def _run_trading_session(self):
             if not broker_continue:
                 return
 
+            # TODO: I think we should remove the OR. Pandas data can have dividends.
+            # Especially if it was saved from yahoo.
             if not has_data_source or (has_data_source and self.broker.data_source.SOURCE != "PANDAS"):
                 self.strategy._update_cash_with_dividends()
 

lumibot/tools/pandas.py

@@ -55,7 +55,7 @@ def print_full_pandas_dataframes():
     pd.set_option('display.width', 1000)
 
 
-def set_pandas_float_precision(precision: int = 5):
+def set_pandas_float_display_precision(precision: int = 5):
     format_str = '{:.' + str(precision) + 'f}'
     pd.set_option('display.float_format', format_str.format)
 

lumibot/tools/polygon_helper.py

@@ -411,6 +411,20 @@ def get_missing_dates(df_all, asset, start, end):
     dates = pd.Series(df_all.index.date).unique()
     missing_dates = sorted(set(trading_dates) - set(dates))
 
+    # TODO: This code works AFAIK, But when i enable it the tests for "test_polygon_missing_day_caching" and
+    # i don't know why nor how to fix this code or the tests. So im leaving it disabled for now. If you have problems
+    # with NANs in cached polygon data, you can try to enable this code and fix the tests.
+
+    # # Find any dates with nan values in the df_all DataFrame
+    # missing_dates += df_all[df_all.isnull().all(axis=1)].index.date.tolist()
+    #
+    # # make sure the dates are unique
+    # missing_dates = list(set(missing_dates))
+    # missing_dates.sort()
+    #
+    # # finally, filter out any dates that are not in start/end range (inclusive)
+    # missing_dates = [d for d in missing_dates if start.date() <= d <= end.date()]
+
     return missing_dates
 
 

setup.py

@@ -47,7 +47,7 @@
         "appdirs",
         "pyarrow",
         "tqdm",
-        "lumiwealth-tradier>=0.1.12",
+        "lumiwealth-tradier>=0.1.14",
         "pytz",
         "psycopg2-binary",
         "exchange_calendars>=4.5.2",

tests/test_drift_rebalancer.py

@@ -13,11 +13,11 @@
 from lumibot.backtesting import BacktestingBroker, YahooDataBacktesting, PandasDataBacktesting
 from lumibot.strategies.strategy import Strategy
 from tests.fixtures import pandas_data_fixture
-from lumibot.tools import print_full_pandas_dataframes, set_pandas_float_precision
+from lumibot.tools import print_full_pandas_dataframes, set_pandas_float_display_precision
 from lumibot.entities import Order
 
 print_full_pandas_dataframes()
-set_pandas_float_precision(precision=5)
+set_pandas_float_display_precision(precision=5)
 
 
 class MockStrategyWithDriftCalculationLogic(Strategy):