Add useBaseAddress to OptionTypeInfo

HISTORY.md

@@ -15,6 +15,7 @@
 * LOG Reporting: add reporting capabilities to the WriteController and the WriteBufferManager by saving the Loggers of the dbs which are using them internally and issuing WARN msgs to these Loggers whenever the state of the WC and WBM changes in regards to delaying (#556).
 * Enable speedb features: Use Scoped Pinning Policy in Enable speedb feature (#459).
 * sst_dump: display metaindex_handle and the index_handle's offset and size in footer information (#404).
+* Added a kUseBaseAddress flag and GetBaseOffset flag to OptionTypeInfo.  If this flag is set and a function is used for processing options, the function is passed the base address of the struct rather than the specific field (#397)
 * Static Pinning: Set the default for mid-percent capacity threshold in scoped pinning policy to 70 (#689).
 * db_bench: Add support for individual scoped pinning policy parameters (#687).
 
@@ -435,6 +436,10 @@ Based on RocksDB 7.2.2
 * Fix a bug in key range overlap checking with concurrent compactions when user-defined timestamp is enabled. User-defined timestamps should be EXCLUDED when checking if two ranges overlap.
 * Fixed a bug where the blob cache prepopulating logic did not consider the secondary cache (see #10603).
 * Fixed the rocksdb.num.sst.read.per.level, rocksdb.num.index.and.filter.blocks.read.per.level and rocksdb.num.level.read.per.multiget stats in the MultiGet coroutines
+* Fix a bug in io_uring_prep_cancel in AbortIO API for posix which expects sqe->addr to match with read request submitted and wrong paramter was being passed.
+* Fixed a regression in iterator performance when the entire DB is a single memtable introduced in #10449. The fix is in #10705 and #10716.
+* Fix a bug in io_uring_prep_cancel in AbortIO API for posix which expects sqe->addr to match with read request submitted and wrong paramter was being passed.
+* Fixed a regression in iterator performance when the entire DB is a single memtable introduced in #10449. The fix is in #10705 and #10716.
 
 ### Public API changes
 * Add `rocksdb_column_family_handle_get_id`, `rocksdb_column_family_handle_get_name` to get name, id of column family in C API

db/column_family.cc

@@ -51,6 +51,7 @@
 #include "port/port.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/table.h"
+#include "rocksdb/utilities/options_type.h"
 #include "rocksdb/write_controller.h"
 #include "table/merging_iterator.h"
 #include "util/autovector.h"
@@ -221,8 +222,8 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
   size_t clamp_max = std::conditional<
       sizeof(size_t) == 4, std::integral_constant<size_t, 0xffffffff>,
       std::integral_constant<uint64_t, 64ull << 30>>::type::value;
-  ClipToRange(&result.write_buffer_size, (static_cast<size_t>(64)) << 10,
-              clamp_max);
+  OptionTypeInfo::ClipToRange(&result.write_buffer_size,
+                              (static_cast<size_t>(64)) << 10, clamp_max);
   // if user sets arena_block_size, we trust user to use this value. Otherwise,
   // calculate a proper value from writer_buffer_size;
   if (result.arena_block_size <= 0) {

db/db_impl/db_impl.h

@@ -2852,13 +2852,6 @@ extern uint64_t FindMinPrepLogReferencedByMemTable(
     VersionSet* vset,
     const autovector<const autovector<MemTable*>*>& memtables_to_flush);
 
-// Fix user-supplied options to be reasonable
-template <class T, class V>
-static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
-  if (static_cast<V>(*ptr) > maxvalue) *ptr = maxvalue;
-  if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
-}
-
 inline Status DBImpl::FailIfCfHasTs(
     const ColumnFamilyHandle* column_family) const {
   column_family = column_family ? column_family : DefaultColumnFamily();

db/db_impl/db_impl_open.cc

@@ -35,6 +35,7 @@
 #include "monitoring/persistent_stats_history.h"
 #include "options/options_helper.h"
 #include "rocksdb/table.h"
+#include "rocksdb/utilities/options_type.h"
 #include "rocksdb/wal_filter.h"
 #include "test_util/sync_point.h"
 #include "util/rate_limiter.h"
@@ -64,7 +65,7 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src,
     if (max_max_open_files == -1) {
       max_max_open_files = 0x400000;
     }
-    ClipToRange(&result.max_open_files, 20, max_max_open_files);
+    OptionTypeInfo::ClipToRange(&result.max_open_files, 20, max_max_open_files);
     TEST_SYNC_POINT_CALLBACK("SanitizeOptions::AfterChangeMaxOpenFiles",
                              &result.max_open_files);
   }

include/rocksdb/utilities/options_type.h

@@ -115,14 +115,15 @@ enum class OptionTypeFlags : uint32_t {
   kCompareLoose = ConfigOptions::kSanityLevelLooselyCompatible,
   kCompareExact = ConfigOptions::kSanityLevelExactMatch,
 
-  kMutable = 0x0100,         // Option is mutable
-  kRawPointer = 0x0200,      // The option is stored as a raw pointer
-  kShared = 0x0400,          // The option is stored as a shared_ptr
-  kUnique = 0x0800,          // The option is stored as a unique_ptr
-  kAllowNull = 0x1000,       // The option can be null
-  kDontSerialize = 0x2000,   // Don't serialize the option
-  kDontPrepare = 0x4000,     // Don't prepare or sanitize this option
-  kStringNameOnly = 0x8000,  // The option serializes to a name only
+  kMutable = 0x00100,         // Option is mutable
+  kRawPointer = 0x00200,      // The option is stored as a raw pointer
+  kShared = 0x00400,          // The option is stored as a shared_ptr
+  kUnique = 0x00800,          // The option is stored as a unique_ptr
+  kAllowNull = 0x01000,       // The option can be null
+  kDontSerialize = 0x02000,   // Don't serialize the option
+  kDontPrepare = 0x04000,     // Don't prepare or sanitize this option
+  kStringNameOnly = 0x08000,  // The option serializes to a name only
+  kUseBaseAddress = 0x10000,  // Pass the base (instead of offset) to functions
 };
 
 inline OptionTypeFlags operator|(const OptionTypeFlags& a,
@@ -251,6 +252,7 @@ using ValidateFunc = std::function<Status(
     const std::string& /*name*/, const void* /*addr*/)>;
 
 class OptionProperties : public std::unordered_map<std::string, std::string> {};
+
 // A struct for storing constant option information such as option name,
 // option type, and offset.
 class OptionTypeInfo {
@@ -324,13 +326,7 @@ class OptionTypeInfo {
         // @return InvalidArgument if the value is not found in the map
         [map](const ConfigOptions&, const std::string& name,
               const std::string& value, void* addr) {
-          if (map == nullptr) {
-            return Status::NotSupported("No enum mapping ", name);
-          } else if (ParseEnum<T>(*map, value, static_cast<T*>(addr))) {
-            return Status::OK();
-          } else {
-            return Status::InvalidArgument("No mapping for enum ", name);
-          }
+          return StringToEnum<T>(name, map, value, static_cast<T*>(addr));
         });
     info.SetSerializeFunc(
         // Uses the map argument to convert the input enum into
@@ -340,14 +336,8 @@ class OptionTypeInfo {
         // @return InvalidArgument if the enum is not found in the map
         [map](const ConfigOptions&, const std::string& name, const void* addr,
               std::string* value) {
-          if (map == nullptr) {
-            return Status::NotSupported("No enum mapping ", name);
-          } else if (SerializeEnum<T>(*map, (*static_cast<const T*>(addr)),
-                                      value)) {
-            return Status::OK();
-          } else {
-            return Status::InvalidArgument("No mapping for enum ", name);
-          }
+          return EnumToString<T>(name, map, *static_cast<const T*>(addr),
+                                 value);
         });
     info.SetEqualsFunc(
         // Casts addr1 and addr2 to the enum type and returns true if
@@ -736,6 +726,26 @@ class OptionTypeInfo {
     return static_cast<char*>(base) + offset_;
   }
 
+  // Returns either the base or the base+offset address,
+  // depending on the kUseBaseAddress flag
+  template <typename T>
+  const void* GetBaseOffset(const void* base, const std::function<T>& f) const {
+    if (f != nullptr && IsEnabled(OptionTypeFlags::kUseBaseAddress)) {
+      return base;
+    } else {
+      return GetOffset(base);
+    }
+  }
+
+  template <typename T>
+  void* GetBaseOffset(void* base, const std::function<T>& f) const {
+    if (f != nullptr && IsEnabled(OptionTypeFlags::kUseBaseAddress)) {
+      return base;
+    } else {
+      return GetOffset(base);
+    }
+  }
+
   template <typename T>
   const T* GetOffsetAs(const void* base) const {
     const void* addr = GetOffset(base);
@@ -825,6 +835,33 @@ class OptionTypeInfo {
   Status Validate(const DBOptions& db_opts, const ColumnFamilyOptions& cf_opts,
                   const std::string& name, const void* opt_ptr) const;
 
+  template <typename E>
+  static Status StringToEnum(
+      const std::string& name,
+      const std::unordered_map<std::string, E>* const map,
+      const std::string& value, E* e) {
+    if (map == nullptr) {
+      return Status::NotSupported("No enum mapping ", name);
+    } else if (ParseEnum(*map, value, e)) {
+      return Status::OK();
+    } else {
+      return Status::InvalidArgument("No mapping for enum ", name);
+    }
+  }
+  template <typename E>
+  static Status EnumToString(
+      const std::string& name,
+      const std::unordered_map<std::string, E>* const map, E e,
+      std::string* value) {
+    if (map == nullptr) {
+      return Status::NotSupported("No enum mapping ", name);
+    } else if (SerializeEnum(*map, e, value)) {
+      return Status::OK();
+    } else {
+      return Status::InvalidArgument("No mapping for enum ", name);
+    }
+  }
+
   // Parses the input opts_map according to the type_map for the opt_addr
   // For each name-value pair in opts_map, find the corresponding name in
   // type_map If the name is found:
@@ -948,6 +985,22 @@ class OptionTypeInfo {
   constexpr static const char* kIdPropName() { return "id"; }
   constexpr static const char* kIdPropSuffix() { return ".id"; }
 
+  // Fix user-supplied options to be reasonable
+  template <class T, class V>
+  static void ClipToMin(T* ptr, V minvalue) {
+    if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
+  }
+  template <class T, class V>
+  static void ClipToMax(T* ptr, V maxvalue) {
+    if (static_cast<V>(*ptr) > maxvalue) *ptr = maxvalue;
+  }
+
+  template <class T, class V>
+  static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
+    ClipToMin(ptr, minvalue);
+    ClipToMax(ptr, maxvalue);
+  }
+
   static std::string MakePrefix(const std::string& prefix,
                                 const std::string& name) {
     if (prefix.empty()) {