repo-sync-2023-04-04T19:01:45+0800 (#56)

* repo-sync-2023-04-04T19:01:45+0800

* Update reinstall.sh

* Update WORKSPACE

.bazelrc

@@ -9,6 +9,7 @@ build --host_cxxopt=-std=c++17
 build --linkopt -fvisibility=hidden
 build --linkopt -lm
 
+
 # platform specific config
 # Bazel will automatic pick platform config since we have enable_platform_specific_config set
 

CHANGELOGS.md

@@ -10,6 +10,16 @@
 
 > please add your unreleased change here.
 
+## [0.4.2]
+- [Feature] Modify api: FeatureWiseBucketSum now support cumulative sum option.
+- [Feature] Modify public key to string message.
+
+## [0.4.1]
+- [Feature] New api: PMatrix/CMatrix add FeatureWiseBucketSum api for better
+  performance
+- [Feature] New api: pylib extension add TreePredict api for performance
+- [Optimize] Optimize CI.
+
 ## [0.4.0]
 
 - [Feature] Add Okamoto–Uchiyama cryptosystem

WORKSPACE

@@ -7,7 +7,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 SECRETFLOW_GIT = "https://github.com/secretflow"
 
-YACL_COMMIT_ID  = "a61d14be36d902435d20002572f21cb4e598b468"
+YACL_COMMIT_ID  = "289794e823f5f664742194e51ce00f1e30017e44"
 
 git_repository(
     name = "yacl",
@@ -41,15 +41,6 @@ rules_foreign_cc_dependencies(
 
 #### for python ####
 
-http_archive(
-    name = "rules_python",
-    sha256 = "8c15896f6686beb5c631a4459a3aa8392daccaab805ea899c9d14215074b60ef",
-    strip_prefix = "rules_python-0.17.3",
-    urls = [
-        "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.17.3.tar.gz",
-    ],
-)
-
 # Python binding.
 http_archive(
     name = "pybind11_bazel",

build_wheel_entrypoint.sh

@@ -0,0 +1,26 @@
+#! /bin/bash
+#
+# Copyright 2022 Ant Group Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+BAZEL_EXTRA_FLAGS=""
+
+
+bazel build --jobs=8 --ui_event_filters=-info,-debug,-warning //heu/pylib -c opt $BAZEL_EXTRA_FLAGS
+heu_wheel_name=$(<bazel-bin/heu/pylib/pylib.name)
+heu_wheel_path="bazel-bin/heu/pylib/${heu_wheel_name}"
+
+cp $heu_wheel_path ./
+cp bazel-bin/heu/pylib/pylib.name ./

docs/requirements.txt

@@ -8,5 +8,4 @@ sphinx-inline-tabs
 sf-heu
 
 # theme
-# if sf-pydata-sphinx-theme install fails, please replace it to pydata-sphinx-theme
-sf-pydata-sphinx-theme
+pydata-sphinx-theme

heu/library/algorithms/ou/public_key.cc

@@ -44,7 +44,7 @@ void PublicKey::Init() {
 
 std::string PublicKey::ToString() const {
   return fmt::format(
-      "OU public key: n={}[{}bits], G={}[{}bits], H={}[{}bits], "
+      "OU PK: n={}[{}bits], G={}[{}bits], H={}[{}bits], "
       "max_plaintext={}[~{}bits]",
       n_.ToHexString(), n_.BitCount(), capital_g_.ToHexString(),
       capital_g_.BitCount(), capital_h_.ToHexString(), capital_h_.BitCount(),

heu/library/algorithms/ou/secret_key.h

@@ -46,9 +46,8 @@ class SecretKey : public HeObject<SecretKey> {
   }
 
   [[nodiscard]] std::string ToString() const override {
-    return fmt::format("OU secret key, p={}[{}bits], q={}[{}bits]",
-                       p_.ToHexString(), p_.BitCount(), q_.ToHexString(),
-                       q_.BitCount());
+    return fmt::format("OU SK, p={}[{}bits], q={}[{}bits]", p_.ToHexString(),
+                       p_.BitCount(), q_.ToHexString(), q_.BitCount());
   }
 };
 

heu/library/algorithms/paillier_float/public_key.cc

@@ -30,7 +30,7 @@ void PublicKey::Init() {
 
 std::string PublicKey::ToString() const {
   return fmt::format(
-      "F-paillier public key: n={}[{}bits], max_plaintext={}[~{}bits]",
+      "F-paillier PK: n={}[{}bits], max_plaintext={}[~{}bits]",
       n_.ToHexString(), n_.BitCount(), PlaintextBound().ToHexString(),
       PlaintextBound().BitCount());
 }

heu/library/algorithms/paillier_float/secret_key.cc

@@ -26,7 +26,7 @@ SecretKey::SecretKey(PublicKey pk, MPInt p, MPInt q) : pk_(std::move(pk)) {
 }
 
 std::string SecretKey::ToString() const {
-  return fmt::format("F-paillier secret key: lambda={}[{}bits], x={}[{}bits]",
+  return fmt::format("F-paillier SK: lambda={}[{}bits], x={}[{}bits]",
                      lambda_.ToHexString(), lambda_.BitCount(),
                      x_.ToHexString(), x_.BitCount());
 }

heu/library/algorithms/paillier_ipcl/public_key.cc

@@ -23,7 +23,7 @@ std::string PublicKey::ToString() const {
   ipcl_pubkey_.getN()->num2hex(n);
   ipcl_pubkey_.getHS().num2hex(hs);
   return fmt::format(
-      "IPCL public key: n={}[{}bits], h_s={}, max_plaintext={}[~{}bits]", n,
+      "IPCL PK: n={}[{}bits], h_s={}, max_plaintext={}[~{}bits]", n,
       ipcl_pubkey_.getN()->BitSize(), hs, PlaintextBound().ToHexString(),
       PlaintextBound().BitCount());
 }

heu/library/algorithms/paillier_ipcl/secret_key.cc

@@ -15,10 +15,13 @@ bool SecretKey::operator!=(const SecretKey &other) const {
 }
 
 std::string SecretKey::ToString() const {
-  std::string lambda;
-  ipcl_prikey_.getLambda().num2hex(lambda);
-  return fmt::format("IPCL secret key: lambda={}[{}bits]", lambda,
-                     ipcl_prikey_.getLambda().BitSize());
+  std::string p;
+  ipcl_prikey_.getP()->num2hex(p);
+  std::string q;
+  ipcl_prikey_.getP()->num2hex(q);
+  return fmt::format("IPCL SK: p={}[{}bits], q={}[{}bits]", p,
+                     ipcl_prikey_.getP()->BitSize(), q,
+                     ipcl_prikey_.getQ()->BitSize());
 }
 
 }  // namespace heu::lib::algorithms::paillier_ipcl

heu/library/algorithms/paillier_zahlen/public_key.cc

@@ -41,7 +41,7 @@ void PublicKey::Init() {
 
 std::string PublicKey::ToString() const {
   return fmt::format(
-      "Z-paillier public key: n={}[{}bits], h_s={}, max_plaintext={}[~{}bits]",
+      "Z-paillier PK: n={}[{}bits], h_s={}, max_plaintext={}[~{}bits]",
       n_.ToHexString(), n_.BitCount(), h_s_.ToHexString(),
       PlaintextBound().ToHexString(), PlaintextBound().BitCount());
 }

heu/library/algorithms/paillier_zahlen/secret_key.cc

@@ -50,9 +50,9 @@ MPInt SecretKey::PowModNSquareCrt(const MPInt& base, const MPInt& exp) const {
 }
 
 std::string SecretKey::ToString() const {
-  return fmt::format("Z-paillier secret key: lambda={}[{}bits], μ={}[{}bits]",
-                     lambda_.ToHexString(), lambda_.BitCount(),
-                     mu_.ToHexString(), mu_.BitCount());
+  return fmt::format("Z-paillier SK: p={}[{}bits], q={}[{}bits]",
+                     p_.ToHexString(), p_.BitCount(), q_.ToHexString(),
+                     q_.BitCount());
 }
 
 }  // namespace heu::lib::algorithms::paillier_z

heu/library/numpy/evaluator.cc

@@ -11,7 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 #include "heu/library/numpy/evaluator.h"
 
 #include "fmt/ranges.h"
@@ -359,4 +358,93 @@ T Evaluator::Sum(const DenseMatrix<T>& x) const {
 template phe::Ciphertext Evaluator::Sum(const CMatrix&) const;
 template phe::Plaintext Evaluator::Sum(const PMatrix&) const;
 
+template <typename T>
+DenseMatrix<T> Evaluator::FeatureWiseBucketSum(
+    const DenseMatrix<T>& x, const Eigen::Ref<RowMatrixXd>& order_map,
+    int bucket_num, bool cumsum) const {
+  // the output is a dense matrix of size sum(bucket_sums) * x.shape[1]
+  YACL_ENFORCE(x.cols() > 0 && x.rows() > 0,
+               "you cannot sum an empty tensor, shape={}x{}", x.rows(),
+               x.cols());
+  YACL_ENFORCE_EQ(order_map.rows(), (x.rows()),
+                  "order map and x should have same number of rows.");
+  // assume all rows of order map has length feature num
+  auto total_bucket_num = bucket_num * order_map.cols();
+  auto res = DenseMatrix<T>(total_bucket_num, x.cols());
+  FeatureWiseBucketSumInplace(x, order_map, bucket_num, res, cumsum);
+  return res;
+}
+
+template CMatrix Evaluator::FeatureWiseBucketSum(
+    const CMatrix&, const Eigen::Ref<RowMatrixXd>& order_map, int bucket_num,
+    bool cumsum) const;
+
+template PMatrix Evaluator::FeatureWiseBucketSum(
+    const PMatrix&, const Eigen::Ref<RowMatrixXd>& order_map, int bucket_num,
+    bool cumsum) const;
+
+template <typename T>
+void Evaluator::FeatureWiseBucketSumInplace(
+    const DenseMatrix<T>& x, const Eigen::Ref<RowMatrixXd>& order_map,
+    int bucket_num, DenseMatrix<T>& res, bool cumsum) const {
+  // the output is a dense matrix of size sum(bucket_sums) * x.shape[1]
+  YACL_ENFORCE(x.cols() > 0 && x.rows() > 0,
+               "you cannot sum an empty tensor, shape={}x{}", x.rows(),
+               x.cols());
+  YACL_ENFORCE_EQ(order_map.rows(), x.rows(),
+                  "order map and x should have same number of rows.");
+  // assume all rows of order map has length feature num
+  auto feature_num = static_cast<size_t>(order_map.cols());
+  auto total_bucket_num = bucket_num * order_map.cols();
+
+  YACL_ENFORCE_EQ(total_bucket_num, res.rows());
+  YACL_ENFORCE_EQ(x.cols(), res.cols());
+  T zero = GetZero(x);
+  // could made this parallel
+  for (auto col = 0; col < x.cols(); ++col) {
+    // feature wise calculations, could be made parallel
+    yacl::parallel_for(0, feature_num, 1, [&](int64_t beg_f, int64_t end_f) {
+      for (auto feature_index = beg_f; feature_index < end_f; ++feature_index) {
+        auto start_offset = bucket_num * feature_index;
+        auto bucket_sums = yacl::parallel_reduce<std::vector<T>>(
+            0, x.rows(), 4 * kHeOpGrainSize,
+            [&](int64_t beg, int64_t end) {
+              auto sums = std::vector<T>(bucket_num, zero);
+              for (auto i = beg; i < end; ++i) {
+                phe::Evaluator::AddInplace(&sums[order_map(i, feature_index)],
+                                           x(i, col));
+              }
+              return sums;
+            },
+            // todo: use binary reduce
+            [&](const std::vector<T>& a, const std::vector<T>& b) {
+              auto result = std::vector<T>(bucket_num);
+              for (auto j = 0; j < bucket_num; ++j) {
+                result[j] = phe::Evaluator::Add(a[j], b[j]);
+              }
+              return result;
+            });
+        if (cumsum) {
+          auto cache_sum = zero;
+          for (auto i = 0; i < bucket_num; ++i) {
+            phe::Evaluator::AddInplace(&cache_sum, bucket_sums[i]);
+            res(i + start_offset, col) = cache_sum;
+          }
+        } else {
+          for (auto i = 0; i < bucket_num; ++i) {
+            res(i + start_offset, col) = bucket_sums[i];
+          }
+        }
+      }
+    });
+  }
+}
+
+template void Evaluator::FeatureWiseBucketSumInplace(
+    const CMatrix&, const Eigen::Ref<RowMatrixXd>& order_map, int bucket_num,
+    CMatrix& res, bool cumsum) const;
+
+template void Evaluator::FeatureWiseBucketSumInplace(
+    const PMatrix&, const Eigen::Ref<RowMatrixXd>& order_map, int bucket_num,
+    PMatrix& res, bool cumsum) const;
 }  // namespace heu::lib::numpy

heu/library/numpy/evaluator.h

@@ -18,6 +18,9 @@
 #include "heu/library/phe/phe.h"
 
 namespace heu::lib::numpy {
+using RowMatrixXd = const Eigen::Matrix<int8_t, Eigen::Dynamic, Eigen::Dynamic,
+                                        Eigen::RowMajor>;
+using RowVector = const Eigen::Matrix<int8_t, 1, Eigen::Dynamic>;
 
 class Evaluator : public phe::Evaluator {
  public:
@@ -59,13 +62,30 @@ class Evaluator : public phe::Evaluator {
                  "you cannot select sum an empty tensor, shape={}x{}", x.rows(),
                  x.cols());
 
-    T zero = phe::Evaluator::Sub(x(0, 0), x(0, 0));
+    T zero = GetZero(x);
     const auto& sub = x.GetItem(row_indices, col_indices);
     if (sub.size() == 0) {
       return zero;
     }
 
     return Sum(sub);
   };
+
+  template <typename T>
+  DenseMatrix<T> FeatureWiseBucketSum(const DenseMatrix<T>& x,
+                                      const Eigen::Ref<RowMatrixXd>& order_map,
+                                      int bucket_num,
+                                      bool cumsum = false) const;
+
+  template <typename T>
+  void FeatureWiseBucketSumInplace(const DenseMatrix<T>& x,
+                                   const Eigen::Ref<RowMatrixXd>& order_map,
+                                   int bucket_num, DenseMatrix<T>& res,
+                                   bool cumsum = false) const;
+
+  template <typename T>
+  T GetZero(const DenseMatrix<T>& x) const {
+    return phe::Evaluator::Sub(x(0, 0), x(0, 0));
+  }
 };
 }  // namespace heu::lib::numpy

heu/library/numpy/numpy_test.cc

@@ -14,6 +14,7 @@
 
 #include "heu/library/numpy/numpy.h"
 
+#include "Eigen/Core"
 #include "gtest/gtest.h"
 
 namespace heu::lib::numpy::test {
@@ -200,6 +201,24 @@ TEST_F(NumpyTest, SelectSumWorks) {
             0 + 1 + 30 + 31);
 }
 
+TEST_F(NumpyTest, BinSumWorks) {
+  auto m = GenMatrix(he_kit_.GetSchemaType(), 5, 2);
+  // whole group
+  std::vector<size_t> subgroup_indices({2, 3, 4});
+
+  // 1 feature all in 0th bucket
+  RowMatrixXd order_map({{1, 3, 2, 2, 3, 3, 0, 0, 0, 0},
+                         {2, 3, 0, 1, 0, 1, 2, 3, 0, 1},
+                         {2, 1, 1, 1, 0, 2, 0, 1, 1, 1},
+                         {3, 2, 2, 0, 0, 1, 1, 1, 2, 3},
+                         {0, 0, 2, 2, 3, 3, 0, 3, 2, 2}});
+  auto bucket_num = 5;
+  auto sum = he_kit_.GetEvaluator()->FeatureWiseBucketSum(
+      m.GetItem(subgroup_indices, Eigen::all),
+      order_map(subgroup_indices, Eigen::all), bucket_num)(2, 0);
+  EXPECT_EQ(sum.GetValue<int64_t>(), 4);
+}
+
 class MatmulTest : public ::testing::TestWithParam<std::tuple<int, int, int>> {
  protected:
   HeKit he_kit_ = HeKit(phe::HeKit(phe::SchemaType::ZPaillier, 2048));

heu/pylib/BUILD.bazel

@@ -5,7 +5,7 @@ load("@rules_python//python:packaging.bzl", "py_package", "py_wheel")
 package(default_visibility = ["//visibility:public"])
 
 
-PYPI_HEU_VERSION = "0.4.0b0"
+PYPI_HEU_VERSION = "0.4.2b0"
 
 exports_files(
     [