diff --git a/Debug b/Debug deleted file mode 100644 index 3f7a5b4..0000000 --- a/Debug +++ /dev/null @@ -1,35 +0,0 @@ -[config] -log_level = 1 -http_server_host = "0.0.0.0" -http_server_port = 8080 -market_update_period_millis = 2000 - -[[symbols]] -symbol = "EUR/USD" -tick_size = 0.00001 -tick_scale = 100000 -price = 1.2 -spread = 0.006 -bid_volume = 1000 -ask_volume = 1000 -bar_period_seconds = 60 -history_age_hours = 96 -history_sample_period_millis = 1000 -[symbols.market_simulator] - model = "white-noise" - sigma = 0.1 - -[[symbols]] -symbol = "AUD/USD" -tick_size = 0.00001 -tick_scale = 100000 -price = 0.78 -spread = 0.002 -bid_volume = 500 -ask_volume = 500 -bar_period_seconds = 60 -history_age_hours = 96 -history_sample_period_millis = 1000 -[symbols.market_simulator] - model = "white-noise" - sigma = 0.1 diff --git a/Release b/Release deleted file mode 100644 index b7f2d9b..0000000 --- a/Release +++ /dev/null @@ -1,26 +0,0 @@ -[DEFAULT] -ConnectionType=acceptor -SocketAcceptPort=5001 -SocketReuseAddress=Y -FileStorePath=store -StartTime=00:00:00 -EndTime=00:00:00 -UseDataDictionary=Y -ReconnectInterval=60 -LogoutTimeout=5 -LogonTimeout=30 -ResetOnLogon=Y -ResetOnLogout=Y -ResetOnDisconnect=Y -SendRedundantResendRequests=Y -SocketNodelay=N -ValidateUserDefinedFields=N -ValidateFieldsOutOfOrder=N -ValidateFieldsHaveValues=N -AllowUnknownMsgFields=Y - -[SESSION] -BeginString=FIX.4.4 -SenderCompID=EXECUTOR -TargetCompID=ZORRO_CLIENT -DataDictionary=../spec/FIX44.xml diff --git a/common/common.vcxproj b/common/common.vcxproj index fc44512..ff490dd 100644 --- a/common/common.vcxproj +++ b/common/common.vcxproj @@ -72,11 +72,11 @@ $(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\ - $(Platform)\$(Configuration)\ + $(LibrariesArchitecture)\$(Configuration)\ $(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\ - $(Platform)\$(Configuration)\ + $(LibrariesArchitecture)\$(Configuration)\ $(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\ @@ -101,7 +101,7 @@ true - copy /y "$(ProjectDir)market_config.toml" "$(SolutionDir)$(Configuration)" + copy /y "$(ProjectDir)market_config.toml" "$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\market_config.toml" @@ -125,7 +125,7 @@ true - copy /y "$(ProjectDir)market_config.toml" "$(SolutionDir)$(Configuration)" + copy /y "$(ProjectDir)market_config.toml" "$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\market_config.toml" @@ -144,6 +144,9 @@ true + + copy /y "$(ProjectDir)market_config.toml" "$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\market_config.toml" + @@ -165,6 +168,9 @@ true true + + copy /y "$(ProjectDir)market_config.toml" "$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\market_config.toml" + diff --git a/fix_simulation_server/fix_simulation_server.vcxproj b/fix_simulation_server/fix_simulation_server.vcxproj index 1ae5a9d..236201e 100644 --- a/fix_simulation_server/fix_simulation_server.vcxproj +++ b/fix_simulation_server/fix_simulation_server.vcxproj @@ -107,7 +107,7 @@ $(SolutionDir)$(LibrariesArchitecture)\$(Configuration);$(SolutionDir)third-parties\quickfix\$(LibrariesArchitecture)-$(Configuration)\lib - copy /y "$(ProjectDir)session.cfg" "$(SolutionDir)$(Configuration)" + copy /y "$(ProjectDir)session.cfg" "$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\session.cfg" @@ -131,7 +131,7 @@ $(SolutionDir)$(LibrariesArchitecture)\$(Configuration);$(SolutionDir)third-parties\quickfix\$(LibrariesArchitecture)-$(Configuration)\lib - copy /y "$(ProjectDir)session.cfg" "$(SolutionDir)$(Configuration)" + copy /y "$(ProjectDir)session.cfg" "$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\session.cfg" @@ -150,6 +150,9 @@ $(CoreLibraryDependencies);%(AdditionalDependencies);quickfixd.lib;common.lib $(SolutionDir)$(LibrariesArchitecture)\$(Configuration);$(SolutionDir)third-parties\quickfix\$(LibrariesArchitecture)-$(Configuration)\lib + + copy /y "$(ProjectDir)session.cfg" "$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\session.cfg" + @@ -171,6 +174,9 @@ $(CoreLibraryDependencies);%(AdditionalDependencies);quickfix.lib;common.lib $(SolutionDir)$(LibrariesArchitecture)\$(Configuration);$(SolutionDir)third-parties\quickfix\$(LibrariesArchitecture)-$(Configuration)\lib + + copy /y "$(ProjectDir)session.cfg" "$(SolutionDir)$(LibrariesArchitecture)\$(Configuration)\session.cfg" + diff --git a/fxcm_market_data_server/proxy_server.h b/fxcm_market_data_server/proxy_server.h index d2a8e5f..5ac1ebe 100644 --- a/fxcm_market_data_server/proxy_server.h +++ b/fxcm_market_data_server/proxy_server.h @@ -439,6 +439,7 @@ namespace fxcm { }); // for example http://localhost:8083/ticks?symbol=EUR/USD&from=2024-06-27 00:00:00 + // http://localhost:8083/ticks?symbol=EUR/USD&count=300 server.Get("/ticks", [this](const Request& req, Response& res) { std::string symbol = "nan"; diff --git a/scripts/copy_zorro_scripts_x64.bat b/scripts/copy_zorro_scripts_x64.bat index 1893af8..cb15048 100644 --- a/scripts/copy_zorro_scripts_x64.bat +++ b/scripts/copy_zorro_scripts_x64.bat @@ -24,6 +24,7 @@ ECHO Copy asset files to %ZORRO_HISTORY_DIR%... robocopy %SOURCE% %ZORRO_HISTORY_DIR% "*.csv" > nul ECHO "Copy FIX spec files to %ZorroInstallDir%\Plugin64\spec..." +robocopy "%SCRIPT_DIR%..\spec" "%ZorroInstallDir%\Plugin\spec" robocopy "%SCRIPT_DIR%..\spec" "%ZorroInstallDir%\Plugin64\spec" ECHO Copy plugin config toml file diff --git a/zorro_fxcm_fix_plugin/zorro_fxcm_fix_plugin.cpp b/zorro_fxcm_fix_plugin/zorro_fxcm_fix_plugin.cpp index 499aa1f..4ad9075 100644 --- a/zorro_fxcm_fix_plugin/zorro_fxcm_fix_plugin.cpp +++ b/zorro_fxcm_fix_plugin/zorro_fxcm_fix_plugin.cpp @@ -30,9 +30,10 @@ #define PLUGIN_VERSION 2 #define PLUGIN_NAME "ZorroFXCMFixPlugin" -#define BAR_DUMP_FILE_NAME "Log/bar_dump.csv" -#define BAR_DUMP_FILE_SEP "," -#define BAR_DUMP_FILE_PREC 5 // FX is point precision i.e. 10-5 = PIPS/10 +#define BAR_DUMP_FILE_NAME "Log/bar_history_dump.csv" +#define TICK_DUMP_FILE_NAME "Log/tick_history_dump.csv" +#define DUMP_FILE_SEP "," +#define DUMP_FILE_PREC 5 // FX is point precision i.e. 10-5 = PIPS/10 namespace zorro { @@ -90,8 +91,10 @@ namespace zorro { } int client_order_id = 0; - int internal_order_id = zorro_cfg["internal_order_id_start"].value().value_or(1000);; + int internal_order_id = zorro_cfg["internal_order_id_start"].value().value_or(1000); + bool convert_tick_to_bar_history_request = zorro_cfg["convert_tick_to_bar_history_request"].value().value_or(true); bool dump_bars_to_file = zorro_cfg["dump_bars_to_file"].value().value_or(true); + bool dump_ticks_to_file = zorro_cfg["dump_ticks_to_file"].value().value_or(true); // these come from Zorro when the plugin is started std::string fxcm_login; @@ -499,27 +502,55 @@ namespace zorro { return; } - log::debug<1, true>("write_bars[skip_header={}]: writing {} bars to {}", skip_header, n_ticks, BAR_DUMP_FILE_NAME); + log::debug("write_bars[skip_header={}]: writing {} bars to {}", skip_header, n_ticks, BAR_DUMP_FILE_NAME); if (!skip_header) { - fs << "bar_end" << BAR_DUMP_FILE_SEP - << "open" << BAR_DUMP_FILE_SEP - << "high" << BAR_DUMP_FILE_SEP - << "low" << BAR_DUMP_FILE_SEP - << "close" << BAR_DUMP_FILE_SEP - << "vol[volume]" << BAR_DUMP_FILE_SEP + fs << "bar_end" << DUMP_FILE_SEP + << "open" << DUMP_FILE_SEP + << "high" << DUMP_FILE_SEP + << "low" << DUMP_FILE_SEP + << "close" << DUMP_FILE_SEP + << "vol[volume]" << DUMP_FILE_SEP << "val[spread]" << std::endl; } for (int i = 0; i < n_ticks; ++i, ++ticks) { - fs << zorro_date_to_string(ticks->time) << BAR_DUMP_FILE_SEP - << format_fp(ticks->fOpen, BAR_DUMP_FILE_PREC) << BAR_DUMP_FILE_SEP - << format_fp(ticks->fHigh, BAR_DUMP_FILE_PREC) << BAR_DUMP_FILE_SEP - << format_fp(ticks->fLow, BAR_DUMP_FILE_PREC) << BAR_DUMP_FILE_SEP - << format_fp(ticks->fClose, BAR_DUMP_FILE_PREC) << BAR_DUMP_FILE_SEP - << format_fp(ticks->fVol, BAR_DUMP_FILE_PREC) << BAR_DUMP_FILE_SEP - << format_fp(ticks->fVal, BAR_DUMP_FILE_PREC) + fs << zorro_date_to_string(ticks->time) << DUMP_FILE_SEP + << format_fp(ticks->fOpen, DUMP_FILE_PREC) << DUMP_FILE_SEP + << format_fp(ticks->fHigh, DUMP_FILE_PREC) << DUMP_FILE_SEP + << format_fp(ticks->fLow, DUMP_FILE_PREC) << DUMP_FILE_SEP + << format_fp(ticks->fClose, DUMP_FILE_PREC) << DUMP_FILE_SEP + << format_fp(ticks->fVol, DUMP_FILE_PREC) << DUMP_FILE_SEP + << format_fp(ticks->fVal, DUMP_FILE_PREC) + << std::endl; + } + + fs.close(); + } + + void write_ticks(T1* ticks, int n_ticks, std::ios_base::openmode mode = std::fstream::app) { + bool skip_header = std::filesystem::exists(TICK_DUMP_FILE_NAME); + + std::fstream fs; + fs.open(TICK_DUMP_FILE_NAME, std::fstream::out | mode); + if (!fs.is_open()) + { + log::error("write_ticks: could not open file {}", TICK_DUMP_FILE_NAME); + return; + } + + log::debug("write_ticks[skip_header={}]: writing {} bars to {}", skip_header, n_ticks, TICK_DUMP_FILE_NAME); + + if (!skip_header) { + fs << "time" << DUMP_FILE_SEP + << "fVal[ask>0,bid<0]" + << std::endl; + } + + for (int i = 0; i < n_ticks; ++i, ++ticks) { + fs << zorro_date_to_string(ticks->time) << DUMP_FILE_SEP + << format_fp(ticks->fVal, DUMP_FILE_PREC) << std::endl; } @@ -580,7 +611,7 @@ namespace zorro { } } - // get historical data - note time is in UTC + // get historical bar data - note time is in UTC // http://localhost:8080/bars?symbol=EUR/USD&from=2024-03-30 12:00:00&to=2024-03-30 16:00:00 int get_historical_bars(const char* Asset, const std::string& timeframe, DATE from, DATE to, std::vector>& bars) { auto from_str = zorro_date_to_string(from); @@ -600,7 +631,7 @@ namespace zorro { return res->status; } - // get historical data - note time is in UTC + // get historical tick data - note time is in UTC // http://localhost:8080/ticks?symbol=EUR/USD&from=2024-06-27 00:00:00 // http://localhost:8080/ticks?symbol=EUR/USD&count=1000 int get_historical_ticks(const char* Asset, DATE from, DATE to, int count, std::vector>& quotes) { @@ -1097,14 +1128,26 @@ namespace zorro { * script since the date and time functions will then return the local time instead of UTC, and time zone functions * cannot be used. */ - DLLFUNC int BrokerHistory2(char* asset, DATE t_start, DATE t_end, int n_tick_minutes, int n_ticks, T6* ticks) { + DLLFUNC int BrokerHistory2(char* asset, DATE t_start, DATE t_end, int n_tick_minutes, int n_ticks, T6* zorro_bars) { + auto now = common::get_current_system_clock(); + auto now_zorro = zorro::convert_time_chrono(now); + auto now_str = zorro_date_to_string(now_zorro); + auto to = zorro_date_to_string(t_end); + + log::debug( + "BrokerHistory2: asset={}, t_start={}[{}], t_end={}[{}], n_tick_minutes={}, n_ticks={}, now={}[{}]", + asset, zorro_date_to_string(t_start), t_start, zorro_date_to_string(t_end), t_end, n_tick_minutes, n_ticks, now_str, now_zorro + ); + + if (n_tick_minutes == 0 && convert_tick_to_bar_history_request) { + n_tick_minutes = 1; + } + if (n_tick_minutes > 0) { auto bar_seconds = n_tick_minutes * 60; auto t_bar = bar_seconds / SECONDS_PER_DAY; - auto t_start2 = t_end - n_ticks * t_bar; - auto from = zorro_date_to_string(t_start2); - auto to = zorro_date_to_string(t_end); - auto ticks_start = ticks; + auto t_start_effective = t_end - n_ticks * t_bar; + auto from = zorro_date_to_string(t_start_effective); std::string timeframe = get_timeframe(n_tick_minutes); if (timeframe == "") { @@ -1112,29 +1155,24 @@ namespace zorro { return 0; } - auto now = common::get_current_system_clock(); - auto now_zorro = zorro::convert_time_chrono(now); - auto now_str = zorro_date_to_string(now_zorro); - - log::debug( - "BrokerHistory2 {}: requesting {} ticks bar period {} minutes from {}[{}] to {}[{}] at {}", - asset, n_ticks, n_tick_minutes, from, t_start2, to, t_end, now_str + log::debug( + "BrokerHistory2 {}: requesting {} bars with bar period {} minutes from {}[{}] to {}[{}] at {}", + asset, n_ticks, n_tick_minutes, from, t_start_effective, to, t_end, now_str ); - log::debug("BrokerHistory2: t_start={}, t_start2={}, t_end={}, now_zorro={}", t_start, t_start2, t_end, now_zorro); - std::vector> bars; - auto status = get_historical_bars(asset, timeframe, t_start2, t_end, bars); + auto status = get_historical_bars(asset, timeframe, t_start_effective, t_end, bars); auto success = status == httplib::StatusCode::OK_200; if (!success) { log::error( "BrokerHistory2: get_historical_prices failed status={} Asset={} timeframe={} from={} to={}", - status, asset, timeframe, t_start2, t_end); + status, asset, timeframe, t_start_effective, t_end); return 0; } int count = 0; + auto zorro_bars_start = zorro_bars; for (auto it = bars.rbegin(); it != bars.rend() && count <= n_ticks; ++it) { const auto& bar = *it; DATE start = bar.timestamp; @@ -1150,38 +1188,95 @@ namespace zorro { continue; } - log::debug<5, false>( + log::debug( "[{}] from={} to={} open={:.5f} high={:.5f} low={:.5f} close={:.5f}", count, zorro_date_to_string(start), zorro_date_to_string(end), bar.ask_open, bar.ask_high, bar.ask_low, bar.ask_close ); - ticks->fOpen = static_cast(bar.ask_open); - ticks->fClose = static_cast(bar.ask_close); - ticks->fHigh = static_cast(bar.ask_high); - ticks->fLow = static_cast(bar.ask_low); - ticks->fVol = static_cast(bar.volume); - ticks->fVal = static_cast(bar.ask_close - bar.bid_close); - ticks->time = end; - ++ticks; + zorro_bars->fOpen = static_cast(bar.ask_open); + zorro_bars->fClose = static_cast(bar.ask_close); + zorro_bars->fHigh = static_cast(bar.ask_high); + zorro_bars->fLow = static_cast(bar.ask_low); + zorro_bars->fVol = static_cast(bar.volume); + zorro_bars->fVal = static_cast(bar.ask_close - bar.bid_close); + zorro_bars->time = end; + ++zorro_bars; ++count; } if (dump_bars_to_file) { - write_bars(ticks_start, count); - write_to_file("Log/broker_hist.csv", + write_bars(zorro_bars_start, count); + write_to_file("Log/broker_bar_hist.csv", std::format( "{}, {}, {}, {}, {}, {}, {}, {}", - asset, timeframe, zorro_date_to_string(t_start2), t_start2, zorro_date_to_string(t_end), t_end, n_ticks, count + asset, timeframe, zorro_date_to_string(t_start_effective), t_start_effective, zorro_date_to_string(t_end), t_end, n_ticks, count ), - "asset, timeframe, t_start2, t_start2[DATE], t_end, t_end[DATE], n_ticks, count" + "asset, timeframe, t_start_eff, t_start_eff[DATE], t_end, t_end[DATE], n_ticks, count" ); } return count; } else { - log::error("BrokerHistory2: called with n_tick_minutes=0 but tick data aka quotes not yet integrated"); - return 0; + log::debug( + "BrokerHistory2 {}: requesting {} ticks to {}[{}] at {}", + asset, n_ticks, to, t_end, now_str + ); + + std::vector> quotes; + auto status = get_historical_ticks(asset, 0, t_end, n_ticks, quotes); + auto success = status == httplib::StatusCode::OK_200; + + if (!success) { + log::error( + "BrokerHistory2: get_historical_ticks failed status={} Asset={} to={} n_ticks={}", + status, asset, t_end, n_ticks); + return 0; + } + + int count = 0; + auto zorro_ticks_start = zorro_bars; + for (auto it = quotes.rbegin(); it != quotes.rend() && count <= n_ticks; ++it) { + const auto& quote = *it; + + if (quote.timestamp > t_end || quote.timestamp < t_start) { + log::debug( + "BrokerHistory2 {}: skipping timestamp {} as it is out of bound t_start={} t_end={}", + asset, zorro_date_to_string(quote.timestamp), zorro_date_to_string(t_start), zorro_date_to_string(t_end) + ); + + continue; + } + + log::debug( + "[{}] timestamp={} bid={:.5f} ask={:.5f} spread={:.5f}", + count, zorro_date_to_string(quote.timestamp), quote.bid, quote.ask, quote.ask - quote.bid + ); + + // set all values to the ask and set spread accordingly + zorro_bars->fOpen = static_cast(quote.ask); + zorro_bars->fClose = static_cast(quote.ask); + zorro_bars->fHigh = static_cast(quote.ask); + zorro_bars->fLow = static_cast(quote.ask); + zorro_bars->fVol = static_cast(0); + zorro_bars->fVal = static_cast(quote.ask - quote.bid); + zorro_bars->time = quote.timestamp; + ++zorro_bars; + ++count; + } + + if (dump_ticks_to_file) { + write_bars(zorro_ticks_start, count); + write_to_file("Log/broker_tick_hist.csv", + std::format( + "{}, {}, {}, {}, {}", + asset, zorro_date_to_string(t_end), t_end, n_ticks, count + ), + "asset, t_end, t_end[DATE], n_ticks, count" + ); + } + + return count; } } diff --git a/zorro_fxcm_fix_plugin/zorro_fxcm_fix_plugin_config.toml b/zorro_fxcm_fix_plugin/zorro_fxcm_fix_plugin_config.toml index b06dae5..664af8f 100644 --- a/zorro_fxcm_fix_plugin/zorro_fxcm_fix_plugin_config.toml +++ b/zorro_fxcm_fix_plugin/zorro_fxcm_fix_plugin_config.toml @@ -1,6 +1,9 @@ [zorro] -internal_order_id_start=1000 -dump_bars_to_file = false +internal_order_id_start = 1000 +convert_tick_to_bar_history_request = true +dump_bars_to_file = true +dump_ticks_to_file = true + [log] spdlog_level = "debug" @@ -14,7 +17,7 @@ login_waiting_time_ms = 10000 exec_report_waiting_time_ms = 1000 termination_waiting_time_ms = 2000 -["fxcm"] +[fxcm] connection = "Demo" fxcm_market_data_server_host = "http://localhost" fxcm_market_data_server_port = 8083