Skip to content

Commit

Permalink
Moved LevelCategory and associated to internal_stats.h
Browse files Browse the repository at this point in the history
  • Loading branch information
udi-speedb committed Mar 18, 2024
1 parent 93cd833 commit e20bac2
Show file tree
Hide file tree
Showing 9 changed files with 75 additions and 58 deletions.
3 changes: 2 additions & 1 deletion db/db_block_cache_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,8 @@ class DBBlockCacheTest : public DBTestBase {
for (const auto cfh : cf_handles) {
auto cfh_impl = static_cast<ColumnFamilyHandleImpl*>(cfh);
auto cache_owner_id = cfh_impl->cfd()->GetCacheOwnerId();
for (auto level_cat_idx = 0U; level_cat_idx < pinning::kNumLevelCategories; ++level_cat_idx) {
for (auto level_cat_idx = 0U; level_cat_idx < kNumLevelCategories;
++level_cat_idx) {
total_role_charges_all_cfs +=
actual_stats.charge_per_item_owner[cache_owner_id][role_idx][level_cat_idx];
}
Expand Down
53 changes: 43 additions & 10 deletions db/internal_stats.cc
Original file line number Diff line number Diff line change
Expand Up @@ -799,10 +799,13 @@ std::string InternalStats::CacheEntryRoleStats::CacheOwnerStatsToString(

str << "Block cache [" << cf_name << "]\n";

str << std::left << std::setw(11) << "Role" << " " << std::setw(10) << "Total";
str << std::left << std::setw(11) << "Role"
<< " " << std::setw(10) << "Total";

for (auto level_cat_idx = 0U; level_cat_idx < pinning::kNumLevelCategories; ++level_cat_idx) {
auto level_cat_name = pinning::GetLevelCategoryShortName(pinning::LevelCategory(level_cat_idx));
for (auto level_cat_idx = 0U; level_cat_idx < kNumLevelCategories;
++level_cat_idx) {
auto level_cat_name =
GetLevelCategoryShortName(LevelCategory(level_cat_idx));
str << std::left << std::setw(10) << level_cat_name;
}
str << '\n';
Expand All @@ -813,14 +816,16 @@ std::string InternalStats::CacheEntryRoleStats::CacheOwnerStatsToString(

uint64_t role_total_charge = 0U;
if (cf_charges_per_role_pos != charge_per_item_owner.end()) {
for (auto level_cat_idx = 0U; level_cat_idx < pinning::kNumLevelCategories; ++level_cat_idx) {
for (auto level_cat_idx = 0U; level_cat_idx < kNumLevelCategories;
++level_cat_idx) {
role_total_charge += cf_charges_per_role_pos->second[role_idx][level_cat_idx];
}
}

str << std::left << std::setw(10) << BytesToHumanString(role_total_charge) << " ";

for (auto level_cat_idx = 0U; level_cat_idx < pinning::kNumLevelCategories; ++level_cat_idx) {
for (auto level_cat_idx = 0U; level_cat_idx < kNumLevelCategories;
++level_cat_idx) {
size_t level_charge = 0U;
if (cf_charges_per_role_pos != charge_per_item_owner.end()) {
level_charge = cf_charges_per_role_pos->second[role_idx][level_cat_idx];
Expand Down Expand Up @@ -868,7 +873,8 @@ void InternalStats::CacheEntryRoleStats::CacheOwnerStatsToMap(
auto role = static_cast<CacheEntryRole>(role_idx);
auto total_role_charge = 0U;
if (cache_owner_charges != charge_per_item_owner.end()) {
for (auto level_cat_idx = 0U; level_cat_idx < pinning::kNumLevelCategories; ++level_cat_idx) {
for (auto level_cat_idx = 0U; level_cat_idx < kNumLevelCategories;
++level_cat_idx) {
total_role_charge +=
cache_owner_charges->second[role_idx][level_cat_idx];
}
Expand Down Expand Up @@ -2268,10 +2274,11 @@ void InternalStats::DumpCFStatsNoFileHistogram(bool is_periodic,
const auto& role_counters = cfd_pinning_counters[level_category_idx];
for (auto role_idx = 0U; role_idx < role_counters.size(); ++role_idx) {
if (role_counters[role_idx] > 0U) {
printf("Total-Pinned[%s][%s]=%d\n",
pinning::GetLevelCategoryName(pinning::LevelCategory(level_category_idx)).c_str(),
GetCacheEntryRoleName(CacheEntryRole(role_idx)).c_str(),
(int)role_counters[role_idx]);
printf(
"Total-Pinned[%s][%s]=%d\n",
GetLevelCategoryName(LevelCategory(level_category_idx)).c_str(),
GetCacheEntryRoleName(CacheEntryRole(role_idx)).c_str(),
(int)role_counters[role_idx]);
all_zeros = false;
}
}
Expand Down Expand Up @@ -2305,5 +2312,31 @@ void InternalStats::DumpCFFileHistogram(std::string* value) {
value->append(oss.str());
}

const std::array<std::string, kNumLevelCategories> kLevelCategoryToHyphenString{
{"level-0", "middle-level", "last-level-with-data", "unknown-level"}};

const std::array<std::string, kNumLevelCategories> kLevelCategoryToShortString{
{"L0", "Middle", "Last", "Unknown"}};

std::string GetLevelCategoryName(LevelCategory category) {
return kLevelCategoryToHyphenString[static_cast<size_t>(category)];
}

std::string GetLevelCategoryShortName(LevelCategory category) {
return kLevelCategoryToShortString[static_cast<size_t>(category)];
}

LevelCategory GetLevelCategory(int level, bool is_last_level_with_data) {
if (is_last_level_with_data) {
return LevelCategory::LAST_LEVEL_WITH_DATA;
} else if (level == 0) {
return LevelCategory::LEVEL_0;
} else if (level == kUnknownLevel) {
return LevelCategory::UNKNOWN_LEVEL;
} else {
assert(level > 0);
return LevelCategory::MIDDLE_LEVEL;
}
}

} // namespace ROCKSDB_NAMESPACE
18 changes: 17 additions & 1 deletion db/internal_stats.h
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,21 @@ struct LevelStat {
std::string header_name;
};

enum class LevelCategory {
LEVEL_0 = 0,
MIDDLE_LEVEL = 1,
LAST_LEVEL_WITH_DATA = 2,
UNKNOWN_LEVEL = 3
};

constexpr uint32_t kNumLevelCategories =
static_cast<uint32_t>(LevelCategory::UNKNOWN_LEVEL) + 1;

std::string GetLevelCategoryName(LevelCategory category);
std::string GetLevelCategoryShortName(LevelCategory category);

LevelCategory GetLevelCategory(int level, bool is_last_level_with_data);

struct DBStatInfo {
// This what will be property_name in the flat map returned to the user
std::string property_name;
Expand Down Expand Up @@ -496,7 +511,8 @@ class InternalStats {
uint32_t hash_seed = 0;

std::unordered_map<Cache::ItemOwnerId,
std::array<std::array<size_t, pinning::kNumLevelCategories>, kNumCacheEntryRoles>>
std::array<std::array<size_t, kNumLevelCategories>,
kNumCacheEntryRoles>>
charge_per_item_owner;

void Clear() {
Expand Down
10 changes: 0 additions & 10 deletions include/rocksdb/table_pinning_policy.h
Original file line number Diff line number Diff line change
Expand Up @@ -46,16 +46,6 @@ constexpr uint32_t kNumHierarchyCategories =

std::string GetHierarchyCategoryName(HierarchyCategory category);

enum class LevelCategory { LEVEL_0 = 0, MIDDLE_LEVEL = 1, LAST_LEVEL_WITH_DATA = 2, UNKNOWN_LEVEL = 3 };

constexpr uint32_t kNumLevelCategories =
static_cast<uint32_t>(LevelCategory::UNKNOWN_LEVEL) + 1;

std::string GetLevelCategoryName(LevelCategory category);
std::string GetLevelCategoryShortName(LevelCategory category);

LevelCategory GetLevelCategory(int level, bool is_last_level_with_data);

} // namespace pinning

// Struct that contains information about the table being evaluated for pinning
Expand Down
4 changes: 3 additions & 1 deletion table/block_based/block_based_table_reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
#include "cache/cache_key.h"
#include "db/compaction/compaction_picker.h"
#include "db/dbformat.h"
#include "db/internal_stats.h"
#include "db/pinned_iterators_manager.h"
#include "file/file_prefetch_buffer.h"
#include "file/file_util.h"
Expand Down Expand Up @@ -1399,7 +1400,8 @@ WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::PutDataBlockToCache(
size_t charge = block_holder->ApproximateMemoryUsage();
BlockCacheTypedHandle<TBlocklike>* cache_handle = nullptr;
auto owner_id_with_level = rep_->cache_owner_id;
auto level_category_value = static_cast<int>(pinning::GetLevelCategory(rep_->level, rep_->last_level_with_data));
auto level_category_value = static_cast<int>(
GetLevelCategory(rep_->level, rep_->last_level_with_data));
owner_id_with_level |= (level_category_value << 14);

s = block_cache.InsertFull(cache_key, block_holder.get(), charge,
Expand Down
4 changes: 2 additions & 2 deletions table/block_based/recording_pinning_policy.cc
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ bool RecordingPinningPolicy::PinData(const TablePinningInfo& tpi,
}

auto level_category_idx = static_cast<uint64_t>(
pinning::GetLevelCategory(tpi.level, tpi.is_last_level_with_data));
GetLevelCategory(tpi.level, tpi.is_last_level_with_data));
auto role_idx = static_cast<uint64_t>(role);

std::lock_guard<std::mutex> lock(counters_mutex_);
Expand Down Expand Up @@ -100,7 +100,7 @@ void RecordingPinningPolicy::UnPinData(
return;
}

auto level_category_idx = static_cast<uint64_t>(pinning::GetLevelCategory(
auto level_category_idx = static_cast<uint64_t>(GetLevelCategory(
pinned_entry->level, pinned_entry->is_last_level_with_data));
auto role_idx = static_cast<uint64_t>(pinned_entry->role);

Expand Down
3 changes: 2 additions & 1 deletion table/block_based/recording_pinning_policy.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include <optional>
#include <unordered_map>

#include "db/internal_stats.h"
#include "rocksdb/cache.h"
#include "rocksdb/table_pinning_policy.h"

Expand All @@ -39,7 +40,7 @@ class RecordingPinningPolicy : public TablePinningPolicy {
public:
using PerRolePinnedCounters = std::array<size_t, kNumCacheEntryRoles>;
using OwnerIdPinnedCounters =
std::array<PerRolePinnedCounters, pinning::kNumLevelCategories>;
std::array<PerRolePinnedCounters, kNumLevelCategories>;

struct OwnerIdInfo {
OwnerIdInfo() = default;
Expand Down
11 changes: 6 additions & 5 deletions table/block_based/recording_pinning_policy_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,11 @@ class RecordingPinningPolicyTest : public testing::Test {
};

TEST_F(RecordingPinningPolicyTest, TestPinningUtils) {
ASSERT_EQ(pinning::LevelCategory::UNKNOWN_LEVEL, pinning::GetLevelCategory(kUnknownLevel, false));
ASSERT_EQ(pinning::LevelCategory::LEVEL_0, pinning::GetLevelCategory(0, false));
ASSERT_EQ(pinning::LevelCategory::MIDDLE_LEVEL, pinning::GetLevelCategory(1, false));
ASSERT_EQ(pinning::LevelCategory::LAST_LEVEL_WITH_DATA, pinning::GetLevelCategory(1, true));
ASSERT_EQ(LevelCategory::UNKNOWN_LEVEL,
GetLevelCategory(kUnknownLevel, false));
ASSERT_EQ(LevelCategory::LEVEL_0, GetLevelCategory(0, false));
ASSERT_EQ(LevelCategory::MIDDLE_LEVEL, GetLevelCategory(1, false));
ASSERT_EQ(LevelCategory::LAST_LEVEL_WITH_DATA, GetLevelCategory(1, true));
}

TEST_F(RecordingPinningPolicyTest, Dummy) {
Expand All @@ -84,7 +85,7 @@ TEST_F(RecordingPinningPolicyTest, Dummy) {
ASSERT_EQ(1000U, policy.GetOwnerIdTotalPinnedUsage(item_owner_id));

auto pinned_counters = policy.GetOwnerIdPinnedUsageCounters(item_owner_id);
auto level_idx = static_cast<uint64_t>(pinning::LevelCategory::LEVEL_0);
auto level_idx = static_cast<uint64_t>(LevelCategory::LEVEL_0);
auto role_idx = static_cast<uint64_t>(role);
ASSERT_EQ(1000U, pinned_counters[level_idx][role_idx]);

Expand Down
27 changes: 0 additions & 27 deletions table/block_based/table_pinning_policy.cc
Original file line number Diff line number Diff line change
Expand Up @@ -48,33 +48,6 @@ std::string GetHierarchyCategoryName(HierarchyCategory category) {
return kHierarchyCategoryToHyphenString[static_cast<size_t>(category)];
}

const std::array<std::string, kNumLevelCategories> kLevelCategoryToHyphenString{
{"level-0", "middle-level", "last-level-with-data", "unknown-level"}};

const std::array<std::string, kNumLevelCategories> kLevelCategoryToShortString{
{"L0", "Middle", "Last", "Unknown"}};

std::string GetLevelCategoryName(LevelCategory category) {
return kLevelCategoryToHyphenString[static_cast<size_t>(category)];
}

std::string GetLevelCategoryShortName(LevelCategory category) {
return kLevelCategoryToShortString[static_cast<size_t>(category)];
}

LevelCategory GetLevelCategory(int level, bool is_last_level_with_data) {
if (is_last_level_with_data) {
return LevelCategory::LAST_LEVEL_WITH_DATA;
} else if (level == 0) {
return LevelCategory::LEVEL_0;
} else if (level == kUnknownLevel) {
return LevelCategory::UNKNOWN_LEVEL;
} else {
assert(level > 0);
return LevelCategory::MIDDLE_LEVEL;
}
}

} // namespace pinning

TablePinningInfo::TablePinningInfo(int _level, bool _is_last_level_with_data,
Expand Down

0 comments on commit e20bac2

Please sign in to comment.