Skip to content

Commit

Permalink
Merge pull request #17 from jhonabreul/feature-null-history-for-unsup…
Browse files Browse the repository at this point in the history 
…ported-securities

Return null on unsupported history and data download requests
  • Loading branch information
@jhonabreul
jhonabreul authored Feb 27, 2024
2 parents 9053e2d + 5abbf46 commit 7b2d453
Show file tree
Hide file tree
Showing 6 changed files with 151 additions and 120 deletions.
123 changes: 53 additions & 70 deletions QuantConnect.OandaBrokerage.Tests/OandaBrokerageHistoryProviderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,13 @@
*/

using System;
using System.Linq;
using NodaTime;
using NUnit.Framework;
using QuantConnect.Brokerages.Oanda;
using QuantConnect.Configuration;
using QuantConnect.Data;
using QuantConnect.Data.Market;
using QuantConnect.Lean.Engine.DataFeeds;
using QuantConnect.Lean.Engine.HistoricalData;
using QuantConnect.Logging;
using QuantConnect.Securities;
using Environment = QuantConnect.Brokerages.Oanda.Environment;
Expand All @@ -35,94 +34,78 @@ private static TestCaseData[] TestParameters
{
get
{
TestGlobals.Initialize();
var eurusd = Symbol.Create("EURUSD", SecurityType.Forex, Market.Oanda);

return new[]
{
// valid parameters
new TestCaseData(eurusd, Resolution.Second, Time.OneMinute, false),
new TestCaseData(eurusd, Resolution.Minute, Time.OneHour, false),
new TestCaseData(eurusd, Resolution.Hour, Time.OneDay, false),
new TestCaseData(eurusd, Resolution.Daily, TimeSpan.FromDays(15), false),
new TestCaseData(eurusd, Resolution.Second, Time.OneMinute, TickType.Quote, false),
new TestCaseData(eurusd, Resolution.Minute, Time.OneHour, TickType.Quote, false),
new TestCaseData(eurusd, Resolution.Hour, Time.OneDay, TickType.Quote, false),
new TestCaseData(eurusd, Resolution.Daily, TimeSpan.FromDays(15), TickType.Quote, false),

// invalid resolution, throws "System.ArgumentException : Unsupported resolution: Tick"
new TestCaseData(eurusd, Resolution.Tick, TimeSpan.FromSeconds(15), true),
// invalid resolution, null result
new TestCaseData(eurusd, Resolution.Tick, TimeSpan.FromSeconds(15), TickType.Quote, true),

// invalid period, no error, empty result
new TestCaseData(eurusd, Resolution.Daily, TimeSpan.FromDays(-15), false),
// invalid period, null result
new TestCaseData(eurusd, Resolution.Daily, TimeSpan.FromDays(-15), TickType.Quote, true),

// invalid symbol, no error, empty result
new TestCaseData(Symbol.Create("XYZ", SecurityType.Forex, Market.FXCM), Resolution.Daily, TimeSpan.FromDays(15), false),
// invalid symbol, null result
new TestCaseData(Symbol.Create("XYZ", SecurityType.Forex, Market.FXCM), Resolution.Daily, TimeSpan.FromDays(15), TickType.Quote, true),

// invalid security type, no error, empty result
new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), false),
// invalid security type, null result
new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), TickType.Quote, true),

// invalid market, null result
new TestCaseData(Symbol.Create("EURUSD", SecurityType.Forex, Market.USA), Resolution.Daily, TimeSpan.FromDays(15), TickType.Quote, true),

// invalid tick type, null result
new TestCaseData(eurusd, Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true),
new TestCaseData(eurusd, Resolution.Daily, TimeSpan.FromDays(15), TickType.OpenInterest, true),
};
}
}

[Test, TestCaseSource(nameof(TestParameters))]
public void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, bool throwsException)
public void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType, bool unsupported)
{
TestDelegate test = () =>
var environment = Config.Get("oanda-environment").ConvertTo<Environment>();
var accessToken = Config.Get("oanda-access-token");
var accountId = Config.Get("oanda-account-id");

var brokerage = new OandaBrokerage(null, null, null, environment, accessToken, accountId);

var now = DateTime.UtcNow;
var request = new HistoryRequest(now.Add(-period),
now,
tickType == TickType.Quote ? typeof(QuoteBar) : typeof(TradeBar),
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.EasternStandard),
DateTimeZone.Utc,
Resolution.Minute,
false,
false,
DataNormalizationMode.Adjusted,
tickType);

var history = brokerage.GetHistory(request)?.ToList();

if (unsupported)
{
var environment = Config.Get("oanda-environment").ConvertTo<Environment>();
var accessToken = Config.Get("oanda-access-token");
var accountId = Config.Get("oanda-account-id");

var brokerage = new OandaBrokerage(null, null, null, environment, accessToken, accountId);

var historyProvider = new BrokerageHistoryProvider();
historyProvider.SetBrokerage(brokerage);
historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, new DataPermissionManager(), null));

var now = DateTime.UtcNow;

var requests = new[]
{
new HistoryRequest(now.Add(-period),
now,
typeof(QuoteBar),
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.EasternStandard),
DateTimeZone.Utc,
Resolution.Minute,
false,
false,
DataNormalizationMode.Adjusted,
TickType.Quote)
};
Assert.IsNull(history);
return;
}

var history = historyProvider.GetHistory(requests, TimeZones.Utc);
Assert.IsNotNull(history);

foreach (var slice in history)
{
if (resolution == Resolution.Tick)
{
foreach (var tick in slice.Ticks[symbol])
{
Log.Trace("{0}: {1} - {2} / {3}", tick.Time, tick.Symbol, tick.BidPrice, tick.AskPrice);
}
}
else
{
var bar = slice.QuoteBars[symbol];

Log.Trace("{0}: {1} - O={2}, H={3}, L={4}, C={5}", bar.Time, bar.Symbol, bar.Open, bar.High, bar.Low, bar.Close);
}
}

Log.Trace("Data points retrieved: " + historyProvider.DataPointCount);
};

if (throwsException)
foreach (var bar in history.Cast<QuoteBar>())
{
Assert.Throws<ArgumentException>(test);
}
else
{
Assert.DoesNotThrow(test);
Log.Trace("{0}: {1} - O={2}, H={3}, L={4}, C={5}", bar.Time, bar.Symbol, bar.Open, bar.High, bar.Low, bar.Close);
}

Log.Trace("Data points retrieved: " + history.Count);
}
}
}
17 changes: 11 additions & 6 deletions QuantConnect.OandaBrokerage.Tests/OandaBrokerageTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,17 @@ protected override IBrokerage CreateBrokerage(IOrderProvider orderProvider, ISec
/// <summary>
/// Provides the data required to test each order type in various cases
/// </summary>
public static TestCaseData[] OrderParameters => new[]
public static IEnumerable<TestCaseData> OrderParameters
{
new TestCaseData(new MarketOrderTestParameters(Symbol.Create("EURUSD", SecurityType.Forex, Market.Oanda))).SetName("MarketOrder"),
new TestCaseData(new LimitOrderTestParameters(Symbol.Create("EURUSD", SecurityType.Forex, Market.Oanda), 5m, 0.32m)).SetName("LimitOrder"),
new TestCaseData(new StopMarketOrderTestParameters(Symbol.Create("EURUSD", SecurityType.Forex, Market.Oanda), 5m, 0.32m)).SetName("StopMarketOrder")
};
get
{
TestGlobals.Initialize();

yield return new TestCaseData(new MarketOrderTestParameters(Symbol.Create("EURUSD", SecurityType.Forex, Market.Oanda))).SetName("MarketOrder");
yield return new TestCaseData(new LimitOrderTestParameters(Symbol.Create("EURUSD", SecurityType.Forex, Market.Oanda), 5m, 0.32m)).SetName("LimitOrder");
yield return new TestCaseData(new StopMarketOrderTestParameters(Symbol.Create("EURUSD", SecurityType.Forex, Market.Oanda), 5m, 0.32m)).SetName("StopMarketOrder");
}
}

/// <summary>
/// Gets the symbol to be traded, must be shortable
Expand Down Expand Up @@ -107,7 +112,7 @@ public void ValidateMarketOrders()
{
order.Status = orderEvent.Status;
orders[order.Id] = order;
}
}
}
};
oanda.OrdersStatusChanged += orderStatusChangedCallback;
Expand Down
43 changes: 9 additions & 34 deletions QuantConnect.OandaBrokerage.ToolBox/OandaDataDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,32 +70,19 @@ public SecurityType GetSecurityType(string symbol)
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <param name="parameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
public IEnumerable<BaseData> Get(DataDownloaderGetParameters parameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;

if (tickType != TickType.Quote)
if (!_brokerage.IsValidHistoryRequest(parameters.Symbol, parameters.StartUtc, parameters.EndUtc, parameters.Resolution, parameters.TickType))
{
yield break;
return null;
}

if (!_symbolMapper.IsKnownLeanSymbol(symbol))
throw new ArgumentException("Invalid symbol requested: " + symbol.Value);

if (resolution == Resolution.Tick)
throw new NotSupportedException("Resolution not available: " + resolution);

if (symbol.ID.SecurityType != SecurityType.Forex && symbol.ID.SecurityType != SecurityType.Cfd)
throw new NotSupportedException("SecurityType not available: " + symbol.ID.SecurityType);

if (endUtc < startUtc)
throw new ArgumentException("The end date must be greater or equal than the start date.");
var symbol = parameters.Symbol;
var resolution = parameters.Resolution;
var startUtc = parameters.StartUtc;
var endUtc = parameters.EndUtc;

var barsTotalInPeriod = new List<QuoteBar>();
var barsToSave = new List<QuoteBar>();
Expand Down Expand Up @@ -162,20 +149,8 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
barsTotalInPeriod.AddRange(barsToSave);
}

switch (resolution)
{
case Resolution.Second:
case Resolution.Minute:
case Resolution.Hour:
case Resolution.Daily:
foreach (var bar in LeanData.AggregateQuoteBars(barsTotalInPeriod, symbol, resolution.ToTimeSpan()))
{
yield return bar;
}
break;
}
return LeanData.AggregateQuoteBars(barsTotalInPeriod, symbol, resolution.ToTimeSpan());
}


/// <summary>
/// Groups a list of bars into a dictionary keyed by date
Expand Down
4 changes: 4 additions & 0 deletions QuantConnect.OandaBrokerage.ToolBox/OandaDownloaderProgram.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ public static void OandaDownloader(IList<string> tickers, string resolution, Dat
var symbol = Symbol.Create(ticker, securityType, market);

var data = downloader.Get(new DataDownloaderGetParameters(symbol, castResolution, startDate, endDate, TickType.Quote));
if (data == null)
{
continue;
}

if (allResolutions)
{
Expand Down
70 changes: 67 additions & 3 deletions QuantConnect.OandaBrokerage/OandaBrokerage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ public class OandaBrokerage : Brokerage, IDataQueueHandler
private OandaRestApiBase _api;
private bool _isInitialized;

private bool _unsupportedAssetForHistoryLogged;
private bool _unsupportedResolutionForHistoryLogged;
private bool _unsupportedTickTypeForHistoryLogged;
private bool _invalidTimeRangeHistoryLogged;

/// <summary>
/// The maximum number of bars per historical data request
/// </summary>
Expand Down Expand Up @@ -220,12 +225,71 @@ public override bool CancelOrder(Order order)
/// <returns>An enumerable of bars covering the span specified in the request</returns>
public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
{
if (!_symbolMapper.IsKnownLeanSymbol(request.Symbol))
if (!IsValidHistoryRequest(request.Symbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, request.TickType))
{
return null;
}

return GetHistoryImpl(request);
}

/// <summary>
/// Validates the historical data request parameters
/// </summary>
/// <param name="symbol">The asset symbol data is being requested for</param>
/// <param name="startTimeUtc">The UTC start time of the request</param>
/// <param name="endTimeUtc">The UTC end time of the request</param>
/// <param name="resolution">The resolution of the requested data</param>
/// <param name="tickType">The tick type of the requested data</param>
/// <returns>Whether the parameters are valid for a history request</returns>
public bool IsValidHistoryRequest(Symbol symbol, DateTime startTimeUtc, DateTime endTimeUtc, Resolution resolution, TickType tickType)
{
if (!_api.CanSubscribe(symbol) || !_symbolMapper.IsKnownLeanSymbol(symbol))
{
if (!_unsupportedAssetForHistoryLogged)
{
Log.Trace($"OandaBrokerage.GetHistory(): Unsupported asset: {symbol}, no history returned");
_unsupportedAssetForHistoryLogged = true;
}
return false;
}

if (resolution == Resolution.Tick)
{
if (!_unsupportedResolutionForHistoryLogged)
{
Log.Trace($"OandaBrokerage.GetHistory(): Unsupported resolution: {resolution}, no history returned");
_unsupportedResolutionForHistoryLogged = true;
}
return false;
}

if (tickType != TickType.Quote)
{
Log.Trace("OandaBrokerage.GetHistory(): Invalid symbol: {0}, no history returned", request.Symbol.Value);
yield break;
if (!_unsupportedTickTypeForHistoryLogged)
{
Log.Trace($"OandaBrokerage.GetHistory(): Unsupported tick type: {tickType}, no history returned");
_unsupportedTickTypeForHistoryLogged = true;
}
return false;
}

if (startTimeUtc >= endTimeUtc)
{
if (!_invalidTimeRangeHistoryLogged)
{
Log.Trace("OandaBrokerage.GetHistory(): The request start date must precede the end date, no history returned.");
_invalidTimeRangeHistoryLogged = true;
}

return false;
}

return true;
}

private IEnumerable<BaseData> GetHistoryImpl(HistoryRequest request)
{
var exchangeTimeZone = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.Oanda, request.Symbol, request.Symbol.SecurityType).TimeZone;

// Oanda only has 5-second bars, we return these for Resolution.Second
Expand Down
14 changes: 7 additions & 7 deletions QuantConnect.OandaBrokerage/OandaRestApiBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,14 +397,14 @@ public void Unsubscribe(SubscriptionDataConfig dataConfig)
/// <summary>
/// Returns true if this brokerage supports the specified symbol
/// </summary>
private bool CanSubscribe(Symbol symbol)
public bool CanSubscribe(Symbol symbol)
{
// ignore unsupported security types
if (symbol.ID.SecurityType != SecurityType.Forex && symbol.ID.SecurityType != SecurityType.Cfd)
return false;

// ignore universe symbols
return !symbol.Value.Contains("-UNIVERSE-") && symbol.ID.Market == Market.Oanda;
return
// ignore unsupported security types
(symbol.ID.SecurityType == SecurityType.Forex || symbol.ID.SecurityType == SecurityType.Cfd) &&
// ignore universe symbols
!symbol.Value.Contains("-UNIVERSE-") &&
symbol.ID.Market == Market.Oanda;
}

private bool Refresh()
Expand Down

0 comments on commit 7b2d453

Please sign in to comment.