refactor(rpc): move rpc out of runtime (#1844)

This is a pure refactor patch without any function changes.
The patch is aim to reduce the size and dependencies of rpc library,
mainly move the "rpc" out of "runtime", including:

- Add a test named "dsn_rpc_tests"
- Remove duplicate comments from **/CMakeLists.txt

.github/workflows/lint_and_test_cpp.yaml

@@ -83,7 +83,6 @@ jobs:
     name: IWYU
     needs:
       - cpp_clang_format_linter
-      - cpp_clang_tidy_linter
     runs-on: ubuntu-latest
     env:
       USE_JEMALLOC: OFF
@@ -113,7 +112,6 @@ jobs:
     name: Build Release
     needs:
       - cpp_clang_format_linter
-      - cpp_clang_tidy_linter
     runs-on: ubuntu-latest
     env:
       USE_JEMALLOC: OFF
@@ -162,6 +160,7 @@ jobs:
           - dsn_replica_bulk_load_test
           - dsn_replica_dup_test
           - dsn_replica_split_test
+          - dsn_rpc_tests
           - dsn.replica.test
           - dsn_replication_common_test
           - dsn.replication.simple_kv
@@ -198,7 +197,6 @@ jobs:
     name: Build ASAN
     needs:
       - cpp_clang_format_linter
-      - cpp_clang_tidy_linter
     runs-on: ubuntu-latest
     env:
       USE_JEMALLOC: OFF
@@ -246,6 +244,7 @@ jobs:
           - dsn_replica_bulk_load_test
           - dsn_replica_dup_test
           - dsn_replica_split_test
+          - dsn_rpc_tests
           - dsn.replica.test
           - dsn_replication_common_test
           - dsn.replication.simple_kv
@@ -287,7 +286,6 @@ jobs:
 #    name: Build UBSAN
 #    needs:
 #      - cpp_clang_format_linter
-#      - cpp_clang_tidy_linter
 #    runs-on: ubuntu-latest
 #    env:
 #      USE_JEMALLOC: OFF
@@ -335,6 +333,7 @@ jobs:
 #          - dsn_replica_bulk_load_test
 #          - dsn_replica_dup_test
 #          - dsn_replica_split_test
+#          - dsn_rpc_tests
 #          - dsn.replica.test
 #          - dsn_replication_common_test
 #          - dsn.replication.simple_kv
@@ -369,7 +368,6 @@ jobs:
     name: Build with jemalloc
     needs:
       - cpp_clang_format_linter
-      - cpp_clang_tidy_linter
     runs-on: ubuntu-latest
     env:
       USE_JEMALLOC: ON
@@ -412,7 +410,6 @@ jobs:
     name: Build Release on macOS
     needs:
       - cpp_clang_format_linter
-      - cpp_clang_tidy_linter
     runs-on: macos-12
     steps:
       - name: Install Softwares
@@ -448,7 +445,6 @@ jobs:
     name: Build Debug on CentOS 7
     needs:
       - cpp_clang_format_linter
-      - cpp_clang_tidy_linter
     runs-on: ubuntu-latest
     env:
       USE_JEMALLOC: OFF

.licenserc.yaml

@@ -126,8 +126,6 @@ header:
     - 'src/utils/api_utilities.h'
     - 'src/runtime/app_model.h'
     - 'src/common/json_helper.h'
-    - 'src/runtime/rpc/rpc_stream.h'
-    - 'src/runtime/rpc/serialization.h'
     - 'src/common/serialization_helper/dsn_types.h'
     - 'src/common/serialization_helper/thrift_helper.h'
     - 'src/runtime/serverlet.h'
@@ -163,13 +161,7 @@ header:
     - 'src/runtime/task/future_types.h'
     - 'src/runtime/global_config.h'
     - 'src/common/gpid.h'
-    - 'src/runtime/rpc/group_address.h'
     - 'src/utils/logging_provider.h'
-    - 'src/runtime/rpc/message_parser.h'
-    - 'src/runtime/rpc/network.h'
-    - 'src/runtime/rpc/rpc_address.cpp'
-    - 'src/runtime/rpc/rpc_address.h'
-    - 'src/runtime/rpc/rpc_message.h'
     - 'src/runtime/task/task.h'
     - 'src/runtime/task/task_code.h'
     - 'src/runtime/task/task_queue.h'
@@ -533,26 +525,6 @@ header:
     - 'src/runtime/profiler.cpp'
     - 'src/runtime/profiler_header.h'
     - 'src/runtime/providers.common.cpp'
-    - 'src/runtime/rpc/CMakeLists.txt'
-    - 'src/runtime/rpc/asio_net_provider.cpp'
-    - 'src/runtime/rpc/asio_net_provider.h'
-    - 'src/runtime/rpc/asio_rpc_session.cpp'
-    - 'src/runtime/rpc/asio_rpc_session.h'
-    - 'src/runtime/rpc/dsn_message_parser.cpp'
-    - 'src/runtime/rpc/dsn_message_parser.h'
-    - 'src/runtime/rpc/message_parser.cpp'
-    - 'src/runtime/rpc/message_parser_manager.h'
-    - 'src/runtime/rpc/network.cpp'
-    - 'src/runtime/rpc/network.sim.cpp'
-    - 'src/runtime/rpc/network.sim.h'
-    - 'src/runtime/rpc/raw_message_parser.cpp'
-    - 'src/runtime/rpc/raw_message_parser.h'
-    - 'src/runtime/rpc/rpc_engine.cpp'
-    - 'src/runtime/rpc/rpc_engine.h'
-    - 'src/runtime/rpc/rpc_message.cpp'
-    - 'src/runtime/rpc/rpc_task.cpp'
-    - 'src/runtime/rpc/thrift_message_parser.cpp'
-    - 'src/runtime/rpc/thrift_message_parser.h'
     - 'src/runtime/scheduler.cpp'
     - 'src/runtime/scheduler.h'
     - 'src/runtime/service_api_c.cpp'
@@ -575,21 +547,49 @@ header:
     - 'src/runtime/task/task_tracker.cpp'
     - 'src/runtime/task/task_worker.cpp'
     - 'src/runtime/test/CMakeLists.txt'
-    - 'src/runtime/test/address_test.cpp'
     - 'src/runtime/test/async_call.cpp'
     - 'src/runtime/test/clear.sh'
-    - 'src/runtime/test/config-test-corrupt-message.ini'
     - 'src/runtime/test/config-test-sim.ini'
     - 'src/runtime/test/config-test.ini'
-    - 'src/runtime/test/corrupt_message.cpp'
     - 'src/runtime/test/lpc.cpp'
     - 'src/runtime/test/main.cpp'
-    - 'src/runtime/test/message_utils_test.cpp'
-    - 'src/runtime/test/netprovider.cpp'
     - 'src/runtime/test/pipeline_test.cpp'
-    - 'src/runtime/test/rpc.cpp'
-    - 'src/runtime/test/rpc_holder_test.cpp'
-    - 'src/runtime/test/rpc_message.cpp'
+    - 'src/rpc/CMakeLists.txt'
+    - 'src/rpc/asio_net_provider.cpp'
+    - 'src/rpc/asio_net_provider.h'
+    - 'src/rpc/asio_rpc_session.cpp'
+    - 'src/rpc/asio_rpc_session.h'
+    - 'src/rpc/dsn_message_parser.cpp'
+    - 'src/rpc/dsn_message_parser.h'
+    - 'src/rpc/group_address.h'
+    - 'src/rpc/message_parser.cpp'
+    - 'src/rpc/message_parser.h'
+    - 'src/rpc/message_parser_manager.h'
+    - 'src/rpc/network.cpp'
+    - 'src/rpc/network.h'
+    - 'src/rpc/network.sim.cpp'
+    - 'src/rpc/network.sim.h'
+    - 'src/rpc/raw_message_parser.cpp'
+    - 'src/rpc/raw_message_parser.h'
+    - 'src/rpc/rpc_address.cpp'
+    - 'src/rpc/rpc_address.h'
+    - 'src/rpc/rpc_engine.cpp'
+    - 'src/rpc/rpc_engine.h'
+    - 'src/rpc/rpc_message.cpp'
+    - 'src/rpc/rpc_message.h'
+    - 'src/rpc/rpc_stream.h'
+    - 'src/rpc/rpc_task.cpp'
+    - 'src/rpc/serialization.h'
+    - 'src/rpc/thrift_message_parser.cpp'
+    - 'src/rpc/thrift_message_parser.h'
+    - 'src/rpc/test/address_test.cpp'
+    - 'src/rpc/test/corrupt_message_test.cpp'
+    - 'src/rpc/test/host_port_test.cpp'
+    - 'src/rpc/test/message_utils_test.cpp'
+    - 'src/rpc/test/net_provider_test.cpp'
+    - 'src/rpc/test/rpc_holder_test.cpp'
+    - 'src/rpc/test/rpc_message_test.cpp'
+    - 'src/rpc/test/rpc_test.cpp'
     - 'src/runtime/test/run.sh'
     - 'src/runtime/test/service_api_c.cpp'
     - 'src/runtime/test/sim_lock.cpp'

build_tools/compile_thrift.py

@@ -39,7 +39,7 @@
         "path": "idl",
         "include_fix": {
             "_types.h": {
-                "add": ["\"runtime/rpc/serialization.h\""],
+                "add": ["\"rpc/serialization.h\""],
                 "remove": ["dsn_types.h"]
             },
             "_types.cpp": {

build_tools/recompile_thrift.sh

@@ -29,7 +29,7 @@ rm -rf $TMP_DIR
 mkdir -p $TMP_DIR
 $THIRDPARTY_ROOT/output/bin/thrift --gen cpp:moveable_types -out $TMP_DIR ../idl/rrdb.thrift
 
-sed 's/#include "dsn_types.h"/#include "runtime\/rpc\/rpc_address.h"\n#include "runtime\/rpc\/rpc_host_port.h"\n#include "runtime\/task\/task_code.h"\n#include "utils\/blob.h"/' $TMP_DIR/rrdb_types.h > ../src/include/rrdb/rrdb_types.h
+sed 's/#include "dsn_types.h"/#include "rpc\/rpc_address.h"\n#include "rpc\/rpc_host_port.h"\n#include "runtime\/task\/task_code.h"\n#include "utils\/blob.h"/' $TMP_DIR/rrdb_types.h > ../src/include/rrdb/rrdb_types.h
 sed 's/#include "rrdb_types.h"/#include <rrdb\/rrdb_types.h>/' $TMP_DIR/rrdb_types.cpp > ../src/base/rrdb_types.cpp
 
 rm -rf $TMP_DIR

src/CMakeLists.txt

@@ -50,6 +50,7 @@ add_subdirectory(ranger)
 add_subdirectory(redis_protocol)
 add_subdirectory(remote_cmd)
 add_subdirectory(replica)
+add_subdirectory(rpc)
 add_subdirectory(runtime)
 add_subdirectory(sample)
 add_subdirectory(security)

src/base/pegasus_rpc_types.h

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-#include "runtime/rpc/rpc_holder.h"
+#include "rpc/rpc_holder.h"
 #include "meta_admin_types.h"
 #include "partition_split_types.h"
 #include "duplication_types.h"

src/block_service/block_service.h

@@ -29,11 +29,11 @@
 #include "utils/threadpool_code.h"
 #include "runtime/task/task_code.h"
 #include "common/gpid.h"
-#include "runtime/rpc/serialization.h"
-#include "runtime/rpc/rpc_stream.h"
+#include "rpc/serialization.h"
+#include "rpc/rpc_stream.h"
 #include "runtime/serverlet.h"
 #include "runtime/service_app.h"
-#include "runtime/rpc/rpc_address.h"
+#include "rpc/rpc_address.h"
 #include "common/replication_other_types.h"
 #include "common/replication.codes.h"
 #include <functional>

src/client/partition_resolver.cpp

@@ -31,7 +31,7 @@
 #include "partition_resolver_manager.h"
 #include "runtime/api_layer1.h"
 #include "runtime/api_task.h"
-#include "runtime/rpc/dns_resolver.h"
+#include "rpc/dns_resolver.h"
 #include "runtime/task/task_spec.h"
 #include "utils/fmt_logging.h"
 #include "utils/threadpool_code.h"

src/client/partition_resolver.h

@@ -34,9 +34,9 @@
 #include <vector>
 
 #include "common/gpid.h"
-#include "runtime/rpc/rpc_host_port.h"
-#include "runtime/rpc/rpc_message.h"
-#include "runtime/rpc/serialization.h"
+#include "rpc/rpc_host_port.h"
+#include "rpc/rpc_message.h"
+#include "rpc/serialization.h"
 #include "runtime/task/async_calls.h"
 #include "runtime/task/task.h"
 #include "runtime/task/task_code.h"

src/client/partition_resolver_manager.cpp

@@ -29,8 +29,8 @@
 #include "client/partition_resolver.h"
 #include "partition_resolver_manager.h"
 #include "partition_resolver_simple.h"
-#include "runtime/rpc/group_host_port.h"
-#include "runtime/rpc/rpc_host_port.h"
+#include "rpc/group_host_port.h"
+#include "rpc/rpc_host_port.h"
 #include "utils/autoref_ptr.h"
 #include "utils/fmt_logging.h"
 #include "utils/utils.h"

src/client/partition_resolver_simple.cpp

@@ -34,10 +34,10 @@
 #include "common/gpid.h"
 #include "dsn.layer2_types.h"
 #include "partition_resolver_simple.h"
+#include "rpc/dns_resolver.h"
+#include "rpc/rpc_message.h"
+#include "rpc/serialization.h"
 #include "runtime/api_layer1.h"
-#include "runtime/rpc/dns_resolver.h"
-#include "runtime/rpc/rpc_message.h"
-#include "runtime/rpc/serialization.h"
 #include "runtime/task/async_calls.h"
 #include "runtime/task/task_code.h"
 #include "runtime/task/task_spec.h"

src/client/partition_resolver_simple.h

@@ -34,7 +34,7 @@
 
 #include "client/partition_resolver.h"
 #include "common/serialization_helper/dsn.layer2_types.h"
-#include "runtime/rpc/rpc_host_port.h"
+#include "rpc/rpc_host_port.h"
 #include "runtime/task/task.h"
 #include "runtime/task/task_tracker.h"
 #include "utils/autoref_ptr.h"

src/client/replication_ddl_client.cpp

@@ -49,9 +49,9 @@
 #include "fmt/core.h"
 #include "fmt/format.h"
 #include "meta/meta_rpc_types.h"
+#include "rpc/group_host_port.h"
+#include "rpc/rpc_address.h"
 #include "runtime/api_layer1.h"
-#include "runtime/rpc/group_host_port.h"
-#include "runtime/rpc/rpc_address.h"
 #include "utils/error_code.h"
 #include "utils/fmt_logging.h"
 #include "utils/output_utils.h"

src/client/replication_ddl_client.h

@@ -33,6 +33,7 @@
 #include <map>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <thread>
 #include <utility>
 #include <vector>
@@ -43,11 +44,11 @@
 #include "meta_admin_types.h"
 #include "partition_split_types.h"
 #include "replica_admin_types.h"
-#include "runtime/rpc/dns_resolver.h"
-#include "runtime/rpc/rpc_holder.h"
-#include "runtime/rpc/rpc_host_port.h"
-#include "runtime/rpc/rpc_message.h"
-#include "runtime/rpc/serialization.h"
+#include "rpc/dns_resolver.h"
+#include "rpc/rpc_holder.h"
+#include "rpc/rpc_host_port.h"
+#include "rpc/rpc_message.h"
+#include "rpc/serialization.h"
 #include "runtime/task/async_calls.h"
 #include "runtime/task/task.h"
 #include "runtime/task/task_code.h"
@@ -59,7 +60,6 @@
 #include "utils/flags.h"
 #include "utils/fmt_logging.h"
 #include "utils/ports.h"
-#include <string_view>
 
 DSN_DECLARE_uint32(ddl_client_max_attempt_count);
 DSN_DECLARE_uint32(ddl_client_retry_interval_ms);

src/client/test/ddl_client_test.cpp

@@ -26,8 +26,8 @@
 #include "common/replication.codes.h"
 #include "gtest/gtest.h"
 #include "meta_admin_types.h"
+#include "rpc/rpc_host_port.h"
 #include "runtime/api_layer1.h"
-#include "runtime/rpc/rpc_host_port.h"
 #include "runtime/task/task.h"
 #include "utils/autoref_ptr.h"
 #include "utils/error_code.h"

src/client_lib/pegasus_client_impl.cpp

@@ -23,21 +23,21 @@
 #include <chrono>
 #include <cstdint>
 #include <string>
+#include <string_view>
 #include <type_traits>
 #include <utility>
 
-#include <string_view>
 #include "common/common.h"
 #include "common/replication_other_types.h"
 #include "common/serialization_helper/dsn.layer2_types.h"
 #include "pegasus/client.h"
 #include "pegasus_client_impl.h"
 #include "pegasus_key_schema.h"
 #include "pegasus_utils.h"
+#include "rpc/dns_resolver.h"
+#include "rpc/group_host_port.h"
+#include "rpc/serialization.h"
 #include "rrdb/rrdb.client.h"
-#include "runtime/rpc/dns_resolver.h"
-#include "runtime/rpc/group_host_port.h"
-#include "runtime/rpc/serialization.h"
 #include "runtime/task/async_calls.h"
 #include "runtime/task/task_code.h"
 #include "utils/error_code.h"

src/client_lib/pegasus_client_impl.h

@@ -31,8 +31,8 @@
 #include <unordered_map>
 #include <vector>
 
+#include "rpc/rpc_host_port.h"
 #include "rrdb/rrdb_types.h"
-#include "runtime/rpc/rpc_host_port.h"
 #include "utils/blob.h"
 #include "utils/zlocks.h"
 

src/client_lib/pegasus_scanner_impl.cpp

@@ -34,7 +34,7 @@
 #include "pegasus_key_schema.h"
 #include "rrdb/rrdb.client.h"
 #include "rrdb/rrdb_types.h"
-#include "runtime/rpc/serialization.h"
+#include "rpc/serialization.h"
 #include "utils/blob.h"
 #include "utils/error_code.h"
 #include "utils/fmt_logging.h"