Skip to content

Commit

Permalink
Switching c++14 to c++11, adding boost (#307)
Browse files Browse the repository at this point in the history 
* Switching c++14 to c++11, adding boost
  • Loading branch information
@amnguye
amnguye authored Aug 27, 2019
1 parent 4e10117 commit 1b9db63
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 34 deletions.
29 changes: 23 additions & 6 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,18 +124,23 @@ if(UNIX)
find_package(Threads REQUIRED)
find_package(CURL REQUIRED)
find_package(GnuTLS REQUIRED)
set(CMAKE_CXX_STANDARD 14)
find_package(PkgConfig)
find_package(Boost REQUIRED)
set(Boost_USE_STATIC_LIBS ON)
find_package(Boost COMPONENTS filesystem system REQUIRED)
find_package(Boost COMPONENTS thread REQUIRED)
add_definitions(-std=c++11)
add_definitions(-D_FILE_OFFSET_BITS=64)
set(WARNING "-Wall -Wextra -Werror -pedantic -pedantic-errors")
set(CMAKE_CXX_FLAGS "${CMAKE_THREAD_LIBS_INIT} ${WARNING} ${CMAKE_CXX_FLAGS}")
include_directories(${CMAKE_SOURCE_DIR}/blobfuse ${CMAKE_SOURCE_DIR}/azure-storage-cpp-lite/include ${CURL_INCLUDE_DIRS} ${GNUTLS_INCLUDE_DIR})
include_directories(${CMAKE_SOURCE_DIR}/blobfuse ${CMAKE_SOURCE_DIR}/azure-storage-cpp-lite/include ${CURL_INCLUDE_DIRS} ${GNUTLS_INCLUDE_DIR} ${Boost_INCLUDE_DIR})

set(CMAKE_MACOSX_RPATH ON)

add_executable(blobfuse ${BLOBFUSE_HEADER} ${BLOBFUSE_SOURCE} ${AZURE_STORAGE_HEADER} ${AZURE_STORAGE_SOURCE} blobfuse/main.cpp)

pkg_search_module(UUID REQUIRED uuid)
target_link_libraries(blobfuse ${CURL_LIBRARIES} ${GNUTLS_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${UUID_LIBRARIES} fuse gcrypt)
target_link_libraries(blobfuse ${CURL_LIBRARIES} ${GNUTLS_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${UUID_LIBRARIES} ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} fuse gcrypt)
install(TARGETS blobfuse
PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
DESTINATION bin)
Expand All @@ -156,7 +161,14 @@ if(UNIX)

set(CPACK_GENERATOR "DEB")
set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Microsoft - Azure Storage")
set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "blobfuse 1.1.0 - FUSE adapter for Azure Blob Storage")
set(CPACK_DEBIAN_PACKAGE_DEPENDS "fuse")
set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "blobfuse 1.1.1 - FUSE adapter for Azure Blob Storage")
set(VERSION "1.1.1")
set(CPACK_PACKAGE_VERSION_MAJOR 1)
set(CPACK_PACKAGE_VERSION_MINOR 1)
set(CPACK_PACKAGE_VERSION_RELEASE 1)
set(CPACK_PACKAGE_NAME "blobfuse")
set(CPACK_PACKAGE_VENDOR "Microsoft")
include(CPack)
endif(UNIX)

Expand Down Expand Up @@ -201,9 +213,14 @@ if(INCLUDE_TESTS)


project(blobfusetests)
set(CMAKE_CXX_STANDARD 14)
find_package(Boost REQUIRED)
set(Boost_USE_STATIC_LIBS ON)
find_package(Boost COMPONENTS filesystem system REQUIRED)
find_package(Boost COMPONENTS thread REQUIRED)
add_definitions(-std=c++11)
pkg_search_module(UUID REQUIRED uuid)
include_directories(${Boost_INCLUDE_DIR})
add_executable(blobfusetests ${BLOBFUSE_HEADER} ${BLOBFUSE_SOURCE} ${AZURE_STORAGE_HEADER} ${AZURE_STORAGE_SOURCE} blobfuse/blobfuse.cpp test/cpplitetests.cpp test/attribcachetests.cpp test/attribcachesynchronizationtests.cpp)
target_link_libraries(blobfusetests ${CURL_LIBRARIES} ${GNUTLS_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${UUID_LIBRARIES} fuse gcrypt gmock_main)
target_link_libraries(blobfusetests ${CURL_LIBRARIES} ${GNUTLS_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${UUID_LIBRARIES} ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} fuse gcrypt gmock_main)

endif()
8 changes: 4 additions & 4 deletions azure-storage-cpp-lite/include/blob/blob_client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#include <memory>
#include <string>
#include <mutex>
#include <shared_mutex>
#include <boost/thread/shared_mutex.hpp>
#include <syslog.h>

#include "storage_EXPORTS.h"
Expand Down Expand Up @@ -618,7 +618,7 @@ namespace microsoft_azure { namespace storage {

// A mutex that can be locked in shared or unique mode (reader/writer lock)
// TODO: Consider switching this to be a regular mutex
std::shared_timed_mutex m_mutex;
boost::shared_mutex m_mutex;

// Name of the blob
std::string m_name;
Expand Down Expand Up @@ -650,13 +650,13 @@ namespace microsoft_azure { namespace storage {
{
}

std::shared_ptr<std::shared_timed_mutex> get_dir_item(const std::string& path);
std::shared_ptr<boost::shared_mutex> get_dir_item(const std::string& path);
std::shared_ptr<blob_cache_item> get_blob_item(const std::string& path);

private:
std::map<std::string, std::shared_ptr<blob_cache_item>> blob_cache;
std::mutex blobs_mutex; // Used to protect the blob_cache map itself, not items in the map.
std::map<std::string, std::shared_ptr<std::shared_timed_mutex>> dir_cache;
std::map<std::string, std::shared_ptr<boost::shared_mutex>> dir_cache;
std::mutex dirs_mutex;// Used to protect the dir_cache map itself, not items in the map.
};

Expand Down
48 changes: 24 additions & 24 deletions azure-storage-cpp-lite/src/blob/blob_client_attr_cache_wrapper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@ namespace microsoft_azure {

// Performs a thread-safe map lookup of the input key in the directory map.
// Will create new entries if necessary before returning.
std::shared_ptr<std::shared_timed_mutex> blob_client_attr_cache_wrapper::attribute_cache::get_dir_item(const std::string& path)
std::shared_ptr<boost::shared_mutex> blob_client_attr_cache_wrapper::attribute_cache::get_dir_item(const std::string& path)
{
std::lock_guard<std::mutex> lock(dirs_mutex);
auto iter = dir_cache.find(path);
if(iter == dir_cache.end())
{
auto dir_item = std::make_shared<std::shared_timed_mutex>();
auto dir_item = std::make_shared<boost::shared_mutex>();
dir_cache[path] = dir_item;
return dir_item;
}
Expand Down Expand Up @@ -61,8 +61,8 @@ namespace microsoft_azure {
/// <returns>A response from list_blobs_hierarchical that contains a list of blobs and their details</returns>
list_blobs_hierarchical_response blob_client_attr_cache_wrapper::list_blobs_hierarchical(const std::string &container, const std::string &delimiter, const std::string &continuation_token, const std::string &prefix, int maxresults)
{
std::shared_ptr<std::shared_timed_mutex> dir_mutex = attr_cache.get_dir_item(prefix);
std::unique_lock<std::shared_timed_mutex> uniquelock(*dir_mutex);
std::shared_ptr<boost::shared_mutex> dir_mutex = attr_cache.get_dir_item(prefix);
std::unique_lock<boost::shared_mutex> uniquelock(*dir_mutex);

errno = 0;
list_blobs_hierarchical_response response = m_blob_client_wrapper->list_blobs_hierarchical(container, delimiter, continuation_token, prefix, maxresults);
Expand Down Expand Up @@ -91,7 +91,7 @@ namespace microsoft_azure {
// taken a lock on the directory string.
// It should be fine, there should be no chance of deadlock, as the internal mutex is released before get_blob_item returns, but we should take care when modifying.
std::shared_ptr<blob_client_attr_cache_wrapper::blob_cache_item> cache_item = attr_cache.get_blob_item(response.blobs[i].name);
std::unique_lock<std::shared_timed_mutex> uniquelock(cache_item->m_mutex);
std::unique_lock<boost::shared_mutex> uniquelock(cache_item->m_mutex);
cache_item->m_props = properties;
cache_item->m_confirmed = true;
}
Expand Down Expand Up @@ -124,10 +124,10 @@ namespace microsoft_azure {
{
// Invalidate the cache.
// TODO: consider updating the cache with the new values. Will require modifying cpplite to return info from put_blob.
std::shared_ptr<std::shared_timed_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(blob));
std::shared_ptr<boost::shared_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(blob));
std::shared_ptr<blob_client_attr_cache_wrapper::blob_cache_item> cache_item = attr_cache.get_blob_item(blob);
std::shared_lock<std::shared_timed_mutex> dirlock(*dir_mutex);
std::unique_lock<std::shared_timed_mutex> uniquelock(cache_item->m_mutex);
boost::shared_lock<boost::shared_mutex> dirlock(*dir_mutex);
std::unique_lock<boost::shared_mutex> uniquelock(cache_item->m_mutex);
m_blob_client_wrapper->put_blob(sourcePath, container, blob, metadata);
cache_item->m_confirmed = false;
}
Expand All @@ -143,10 +143,10 @@ namespace microsoft_azure {
{
// Invalidate the cache.
// TODO: consider updating the cache with the new values. Will require modifying cpplite to return info from put_blob.
std::shared_ptr<std::shared_timed_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(blob));
std::shared_ptr<boost::shared_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(blob));
std::shared_ptr<blob_client_attr_cache_wrapper::blob_cache_item> cache_item = attr_cache.get_blob_item(blob);
std::shared_lock<std::shared_timed_mutex> dirlock(*dir_mutex);
std::unique_lock<std::shared_timed_mutex> uniquelock(cache_item->m_mutex);
boost::shared_lock<boost::shared_mutex> dirlock(*dir_mutex);
std::unique_lock<boost::shared_mutex> uniquelock(cache_item->m_mutex);
m_blob_client_wrapper->upload_block_blob_from_stream(container, blob, is, metadata);
cache_item->m_confirmed = false;
}
Expand All @@ -163,10 +163,10 @@ namespace microsoft_azure {
{
// Invalidate the cache.
// TODO: consider updating the cache with the new values. Will require modifying cpplite to return info from put_blob.
std::shared_ptr<std::shared_timed_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(blob));
std::shared_ptr<boost::shared_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(blob));
std::shared_ptr<blob_client_attr_cache_wrapper::blob_cache_item> cache_item = attr_cache.get_blob_item(blob);
std::shared_lock<std::shared_timed_mutex> dirlock(*dir_mutex);
std::unique_lock<std::shared_timed_mutex> uniquelock(cache_item->m_mutex);
boost::shared_lock<boost::shared_mutex> dirlock(*dir_mutex);
std::unique_lock<boost::shared_mutex> uniquelock(cache_item->m_mutex);
m_blob_client_wrapper->upload_file_to_blob(sourcePath, container, blob, metadata, parallel);
cache_item->m_confirmed = false;
}
Expand Down Expand Up @@ -220,21 +220,21 @@ namespace microsoft_azure {
/// Useful if there is reason to suspect the properties may have changed behind the scenes (specifically, if there's a pending copy operation.)</param>
blob_property blob_client_attr_cache_wrapper::get_blob_property(const std::string &container, const std::string &blob, bool assume_cache_invalid)
{
std::shared_ptr<std::shared_timed_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(blob));
std::shared_ptr<boost::shared_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(blob));
std::shared_ptr<blob_client_attr_cache_wrapper::blob_cache_item> cache_item = attr_cache.get_blob_item(blob);
std::shared_lock<std::shared_timed_mutex> dirlock(*dir_mutex);
boost::shared_lock<boost::shared_mutex> dirlock(*dir_mutex);

if (!assume_cache_invalid)
{
std::shared_lock<std::shared_timed_mutex> sharedlock(cache_item->m_mutex);
boost::shared_lock<boost::shared_mutex> sharedlock(cache_item->m_mutex);
if (cache_item->m_confirmed)
{
return cache_item->m_props;
}
}

{
std::unique_lock<std::shared_timed_mutex> uniquelock(cache_item->m_mutex);
std::unique_lock<boost::shared_mutex> uniquelock(cache_item->m_mutex);
errno = 0;
cache_item->m_props = m_blob_client_wrapper->get_blob_property(container, blob);
if (errno != 0)
Expand Down Expand Up @@ -271,10 +271,10 @@ namespace microsoft_azure {
void blob_client_attr_cache_wrapper::delete_blob(const std::string &container, const std::string &blob)
{
// These calls cannot be cached because we do not have a negative cache - blobs in the cache are either valid/confirmed, or unknown (which could be deleted, or not checked on the service.)
std::shared_ptr<std::shared_timed_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(blob));
std::shared_ptr<boost::shared_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(blob));
std::shared_ptr<blob_client_attr_cache_wrapper::blob_cache_item> cache_item = attr_cache.get_blob_item(blob);
std::shared_lock<std::shared_timed_mutex> dirlock(*dir_mutex);
std::unique_lock<std::shared_timed_mutex> uniquelock(cache_item->m_mutex);
boost::shared_lock<boost::shared_mutex> dirlock(*dir_mutex);
std::unique_lock<boost::shared_mutex> uniquelock(cache_item->m_mutex);
m_blob_client_wrapper->delete_blob(container, blob);
cache_item->m_confirmed = false;
}
Expand All @@ -290,10 +290,10 @@ namespace microsoft_azure {
{
// No need to lock on the source, as we're neither modifying nor querying the source blob or its cached content.
// We do need to lock on the destination, because if the start copy operation succeeds we need to invalidate the cached data.
std::shared_ptr<std::shared_timed_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(destBlob));
std::shared_ptr<boost::shared_mutex> dir_mutex = attr_cache.get_dir_item(get_parent_str(destBlob));
std::shared_ptr<blob_client_attr_cache_wrapper::blob_cache_item> cache_item = attr_cache.get_blob_item(destBlob);
std::shared_lock<std::shared_timed_mutex> dirlock(*dir_mutex);
std::unique_lock<std::shared_timed_mutex> uniquelock(cache_item->m_mutex);
boost::shared_lock<boost::shared_mutex> dirlock(*dir_mutex);
std::unique_lock<boost::shared_mutex> uniquelock(cache_item->m_mutex);
errno = 0;
m_blob_client_wrapper->start_copy(sourceContainer, sourceBlob, destContainer, destBlob);
cache_item->m_confirmed = false;
Expand Down

0 comments on commit 1b9db63

Please sign in to comment.