Merge pull request #7 from dsec-capital/de/fxcm_performance

De/fxcm performance

Debug

@@ -0,0 +1,35 @@
+[config]
+log_level = 1
+http_server_host = "0.0.0.0"
+http_server_port = 8080
+market_update_period_millis = 2000
+
+[[symbols]]
+symbol = "EUR/USD"
+tick_size = 0.00001
+tick_scale = 100000
+price = 1.2
+spread = 0.006
+bid_volume = 1000
+ask_volume = 1000
+bar_period_seconds = 60
+history_age_hours = 96
+history_sample_period_millis = 1000
+[symbols.market_simulator]
+	model = "white-noise"
+	sigma = 0.1
+
+[[symbols]]
+symbol = "AUD/USD"
+tick_size = 0.00001
+tick_scale = 100000
+price = 0.78
+spread = 0.002
+bid_volume = 500
+ask_volume = 500
+bar_period_seconds = 60
+history_age_hours = 96
+history_sample_period_millis = 1000
+[symbols.market_simulator]
+	model = "white-noise"
+	sigma = 0.1

README.md

@@ -18,6 +18,7 @@ Contributions, bug reports and constructive feedback are very welcome.
 
 ## Version History
 
+  - v2.1.0 FXCM ForexConnect is accessed through a proxy server, smaller bug fixes, order status messages, 64 bit version 
   - v2.0.0 First FXCM FIX plugin, separated and updated Simulation FIX plugin 
   - v1.0.1 Improving order cancellation and many smaller bug fixing
   - v1.0.0 First official release
@@ -35,11 +36,55 @@ header only third party components:
   - [magic_enum](third_parties/magic_enum)
   - [spdlog](third_parties/spdlog)
 
-Library dependencies are:
 
-  - [QuickFix](third_parties/quickfix) with prebuilt static libraries included
-  - [FXCM ForexConnect SDK](third_parties/fxcm) which requires separate installation, see below
+### Library Dependencies
 
+  - [QuickFix](third_parties/quickfix) with prebuilt static libraries included for x86 and x64
+  - [FXCM ForexConnect SDK](third_parties/fxcm) which requires separate installation 
+   - [x86](http://fxcodebase.com/bin/forexconnect/1.6.5/ForexConnectAPI-1.6.5-win32.exe)
+   - [x64](http://fxcodebase.com/bin/forexconnect/1.6.5/ForexConnectAPI-1.6.5-win64.exe)
+
+
+### Environtment Configuration
+
+The environment variable `%ZorroInstallDir%` has to point to the Zorro installation 
+
+```
+echo %ZorroInstallDir%
+C:\zorro\Zorro_2614
+```
+
+
+The FXCM market data server can log tick data and incremental bar updates. The log path 
+can be configured by specifying the environment variable `FXCM_MAKRET_DATA_SERVER_LOG_PATH` 
+
+```
+echo %FXCM_MAKRET_DATA_SERVER_LOG_PATH%
+C:\zorro\Zorro_2614\Log\fxcm_market_data_server_
+```
+
+The installation of FXCM ForexConnect SDK sets the following environment variables
+
+```
+echo %FOREXCONNECT_PATH_X64%
+C:\Program Files\Candleworks\ForexConnectAPIx64
+```
+
+respectively
+
+```
+echo %FOREXCONNECT_PATH_X86%
+C:\Program Files (x86)\Candleworks\ForexConnectAPI
+```
+
+Assure that they point to the installed version of the ForexConnect SDK.
+
+In order to use the C++ test script from Zorro, add the path to the Visual Studio build directory to 
+the `Zorro.ini` file:
+
+```
+VCPath = "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build" 
+```
 
 
 ## FXCM FIX Plugin  
@@ -70,6 +115,34 @@ The FXCM FIX client plugin implements all of Zorro's [broker plugin functions](h
 
 More details can be found in [FXCM FIX client application](zorro_fxcm_fix_plugin/application.h). 
 
+## FXCM Market Data Proxy Server
+
+From version 2.1.0 the FXCM ForexConnect API is not directly integrated anymore with the Zorro FIX plugin but is running as a 
+standalone proxy server `fxcm_market_data_server`, exposing various rest endpoints. It can be directly started from Visual Studio 
+and extends the system path with the respective path to the FXCM ForexConnect dlls:
+
+Project Properties fxcm_market_data_server -> Configuration Properties -> Debugging -> Environment 
+```
+Path=$(Path);$(FOREXCONNECT_PATH_X64)\bin
+```
+for platform x64, respectively for x86 or Win32
+```
+Path=$(Path);$(FOREXCONNECT_PATH_X86)\bin
+```
+
+Start `fxcm_market_data_server` with the proper path setting
+
+```
+C:\repos\zorro-fix\x64\Debug>fxcm_market_data_server.exe
+[2024-06-19 20:23:51.874] [combined] [debug] Logging started, logger_name=combined, level=1, cwd=C:\repos\zorro-fix\x64\Debug
+[2024-06-19 20:23:51.899] [combined] [info] session status connecting
+[2024-06-19 20:24:00.263] [combined] [info] session status connected
+[2024-06-19 20:24:00.264] [combined] [info] connected - server ready to accept service requests
+[2024-06-19 20:24:00.266] [combined] [info] server stated on 0.0.0.0:8080
+```
+
+It logs to the screen as well as to the log file `fxcm_proxy_server.log`.
+
 
 ### Installation
 

Release

@@ -0,0 +1,35 @@
+[config]
+log_level = 1
+http_server_host = "0.0.0.0"
+http_server_port = 8080
+market_update_period_millis = 2000
+
+[[symbols]]
+symbol = "EUR/USD"
+tick_size = 0.00001
+tick_scale = 100000
+price = 1.2
+spread = 0.006
+bid_volume = 1000
+ask_volume = 1000
+bar_period_seconds = 60
+history_age_hours = 96
+history_sample_period_millis = 1000
+[symbols.market_simulator]
+	model = "white-noise"
+	sigma = 0.1
+
+[[symbols]]
+symbol = "AUD/USD"
+tick_size = 0.00001
+tick_scale = 100000
+price = 0.78
+spread = 0.002
+bid_volume = 500
+ask_volume = 500
+bar_period_seconds = 60
+history_age_hours = 96
+history_sample_period_millis = 1000
+[symbols.market_simulator]
+	model = "white-noise"
+	sigma = 0.1

common/bar.h

@@ -29,12 +29,40 @@ namespace common {
         double close;
 
         std::string to_string() const {
-            return
-                "end=" + common::to_string(end) + ", " +
-                "open=" + std::to_string(open) + ", " +
-                "high=" + std::to_string(high) + ", " +
-                "low=" + std::to_string(low) + ", " +
-                "close=" + std::to_string(close);
+            std::stringstream ss;
+            ss << "Bar[" <<
+                "end=" << common::to_string(end) << ", " <<
+                "open=" << std::to_string(open) << ", " <<
+                "high=" << std::to_string(high) << ", " <<
+                "low=" << std::to_string(low) << ", " <<
+                "close=" << std::to_string(close) << "]";
+            return ss.str();
+        }
+    };
+
+    template<typename T>
+    class Quote {
+    public:
+        Quote(
+            const T& timestamp,
+            double bid,
+            double ask
+        ) : timestamp(timestamp)
+          , bid(bid)
+          , ask(ask)
+        {}
+
+        T timestamp;
+        double bid;
+        double ask;
+
+        std::string to_string() const {
+            std::stringstream ss;
+            ss << "Quote["
+                "timestamp=" << std::to_string(timestamp) << ", " <<
+                "bid=" << std::to_string(bid) << ", " <<
+                "ask=" << std::to_string(ask) << "]";
+            return ss.str();
         }
     };
 
@@ -76,17 +104,19 @@ namespace common {
         double volume;
 
         std::string to_string() const {
-            return
-                "timestamp=" + std::to_string(timestamp) + ", " +
-                "bid_open=" + std::to_string(bid_open) + ", " +
-                "bid_high=" + std::to_string(bid_high) + ", " +
-                "bid_low=" + std::to_string(bid_low) + ", " +
-                "bid_close=" + std::to_string(bid_close) + ", " +
-                "ask_open=" + std::to_string(ask_open) + ", " +
-                "ask_high=" + std::to_string(ask_high) + ", " +
-                "ask_low=" + std::to_string(ask_low) + ", " +
-                "ask_close=" + std::to_string(ask_close) + ", " +
-                "volume=" + std::to_string(volume);
+            std::stringstream ss;
+            ss << "BidAskBar["
+                "timestamp=" << std::to_string(timestamp) << ", " <<
+                "bid_open=" << std::to_string(bid_open) << ", " <<
+                "bid_high=" << std::to_string(bid_high) << ", " <<
+                "bid_low=" << std::to_string(bid_low) << ", " <<
+                "bid_close=" << std::to_string(bid_close) << ", " <<
+                "ask_open=" << std::to_string(ask_open) << ", " <<
+                "ask_high=" << std::to_string(ask_high) << ", " <<
+                "ask_low=" << std::to_string(ask_low) << ", " <<
+                "ask_close=" << std::to_string(ask_close) << ", " <<
+                "volume=" << std::to_string(volume) << "]";
+            return ss.str();
         }
     };
 }

common/bar_builder.h

@@ -10,6 +10,7 @@
 #include "bar.h"
 #include "utils.h"
 #include "time_utils.h"
+#include "market_data.h"
 
 namespace common {
 

common/common.vcxproj

@@ -70,6 +70,20 @@
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
   <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <OutDir>$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\</OutDir>
+    <IntDir>$(Platform)\$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <OutDir>$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\</OutDir>
+    <IntDir>$(Platform)\$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <OutDir>$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\</OutDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <OutDir>$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\</OutDir>
+  </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
@@ -79,7 +93,7 @@
       <PrecompiledHeader>Use</PrecompiledHeader>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
       <AdditionalIncludeDirectories>$(SolutionDir)third-parties;$(SolutionDir)third-parties\quickfix\$(LibrariesArchitecture)-$(Configuration)\include</AdditionalIncludeDirectories>
-      <LanguageStandard>stdcpp20</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>
@@ -101,7 +115,7 @@
       <PrecompiledHeader>Use</PrecompiledHeader>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
       <AdditionalIncludeDirectories>$(SolutionDir)third-parties;$(SolutionDir)third-parties\quickfix\$(LibrariesArchitecture)-$(Configuration)\include</AdditionalIncludeDirectories>
-      <LanguageStandard>stdcpp20</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>
@@ -122,6 +136,8 @@
       <ConformanceMode>true</ConformanceMode>
       <PrecompiledHeader>Use</PrecompiledHeader>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
+      <AdditionalIncludeDirectories>$(SolutionDir)third-parties;$(SolutionDir)third-parties\quickfix\$(LibrariesArchitecture)-$(Configuration)\include</AdditionalIncludeDirectories>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>
@@ -139,6 +155,8 @@
       <ConformanceMode>true</ConformanceMode>
       <PrecompiledHeader>Use</PrecompiledHeader>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
+      <AdditionalIncludeDirectories>$(SolutionDir)third-parties;$(SolutionDir)third-parties\quickfix\$(LibrariesArchitecture)-$(Configuration)\include</AdditionalIncludeDirectories>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>

common/json.h

@@ -42,6 +42,38 @@ namespace common {
 		return std::make_pair(j, count);
 	}
 
+	inline json to_json(const std::vector<BidAskBar<double>>& bars) {
+		std::vector<double> timestamp;
+		std::vector<double> bid_open, bid_close, bid_high, bid_low, ask_open, ask_close, ask_high, ask_low, volume;
+		int count = 0;
+		for (const auto& bar : bars) {
+			timestamp.emplace_back(bar.timestamp);
+			bid_open.emplace_back(bar.bid_open);
+			bid_high.emplace_back(bar.bid_high);
+			bid_low.emplace_back(bar.bid_low);
+			bid_close.emplace_back(bar.bid_close);
+			ask_open.emplace_back(bar.ask_open);
+			ask_high.emplace_back(bar.ask_high);
+			ask_low.emplace_back(bar.ask_low);
+			ask_close.emplace_back(bar.ask_close);
+			volume.emplace_back(bar.volume);
+		}
+
+		json j;
+		j["timestamp"] = timestamp;
+		j["bid_open"] = bid_open;
+		j["bid_high"] = bid_high;
+		j["bid_low"] = bid_low;
+		j["bid_close"] = bid_close;
+		j["ask_open"] = ask_open;
+		j["ask_high"] = ask_high;
+		j["ask_low"] = ask_low;
+		j["ask_close"] = ask_close;
+		j["volume"] = volume;
+
+		return j;
+	}
+
 	inline void from_json(const json& j, std::map<std::chrono::nanoseconds, Bar>& bars) {
 		auto end = j["end"].get<std::vector<long long>>();
 		auto open = j["open"].get<std::vector<double>>();
@@ -64,6 +96,40 @@ namespace common {
 		}
 	}
 
+	inline void from_json(const json& j, std::vector<BidAskBar<double>>& bars) {
+		auto timestamp = j["timestamp"].get<std::vector<double>>();
+		auto n = timestamp.size();
+
+		auto bid_open = j["bid_open"].get<std::vector<double>>();
+		auto bid_high = j["bid_high"].get<std::vector<double>>();
+		auto bid_low = j["bid_low"].get<std::vector<double>>();
+		auto bid_close = j["bid_close"].get<std::vector<double>>();
+		assert(bid_open.size() == n && bid_high.size() == n && bid_low.size() == n && bid_close.size() == n);
+
+		auto ask_open = j["ask_open"].get<std::vector<double>>();
+		auto ask_high = j["ask_high"].get<std::vector<double>>();
+		auto ask_low = j["ask_low"].get<std::vector<double>>();
+		auto ask_close = j["ask_close"].get<std::vector<double>>();
+		auto volume = j["volume"].get<std::vector<double>>();
+		assert(ask_open.size() == n && ask_high.size() == n && ask_low.size() == n && ask_close.size() == n && volume.size() == n);
+
+		bars.clear();
+		for (size_t i = 0; i < n; ++i) {
+			bars.emplace_back(
+				timestamp[i],
+				bid_open[i],
+				bid_high[i],
+				bid_low[i],
+				bid_close[i],
+				ask_open[i],
+				ask_high[i],
+				ask_low[i],
+				ask_close[i],
+				volume[i]
+				);
+		}
+	}
+
 	inline void to_json(json& j, const Bar& b) {
 		j = json{
 			{"end", b.end.count()},