From 04a8ecf1621705fb009c899fdd088f7e8135789e Mon Sep 17 00:00:00 2001 From: Matthieu Dorier Date: Fri, 10 Jan 2025 14:31:38 +0000 Subject: [PATCH] added more extensive test of the log backend via python --- src/backends/log.cpp | 14 ++++++ tests/python/test-coll-log.py | 90 +++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 tests/python/test-coll-log.py diff --git a/src/backends/log.cpp b/src/backends/log.cpp index 65b25fe..5f8c3a6 100644 --- a/src/backends/log.cpp +++ b/src/backends/log.cpp @@ -115,11 +115,13 @@ class LogDatabase : public DatabaseInterface { if(page_size == 0) { // Get the system's page size page_size = sysconf(_SC_PAGESIZE); + // LCOV_EXCL_START if (page_size == -1) { YOKAN_LOG_ERROR(MARGO_INSTANCE_NULL, "sysconf(_SC_PAGESIZE) failed: %s", strerror(errno)); return Status::IOError; } + // LCOV_EXCL_STOP } // Calculate the start of the page-aligned address @@ -135,9 +137,11 @@ class LogDatabase : public DatabaseInterface { // Call msync on the page-aligned address and adjusted size if (msync((void *)page_start, aligned_size, MS_SYNC) == -1) { + /// LCOV_EXCL_START YOKAN_LOG_ERROR(MARGO_INSTANCE_NULL, "msync failed: %s", strerror(errno)); return Status::IOError; + // LCOV_EXCL_STOP } return Status::OK; @@ -147,42 +151,50 @@ class LogDatabase : public DatabaseInterface { // Open or create the file m_fd = open(m_filename.c_str(), O_RDWR | O_CREAT, 0644); if (m_fd < 0) { + // LCOV_EXCL_START YOKAN_LOG_ERROR(MARGO_INSTANCE_NULL, "Failed to open file %s: %s", m_filename.c_str(), strerror(errno)); return Status::IOError; + // LCOV_EXCL_STOP } // Get size of the file auto size = lseek(m_fd, 0L, SEEK_END); if(size < 0) { + // LCOV_EXCL_START YOKAN_LOG_ERROR(MARGO_INSTANCE_NULL, "lseek failed for file %s: %s", m_filename.c_str(), strerror(errno)); close(m_fd); return Status::IOError; + // LCOV_EXCL_STOP } lseek(m_fd, 0L, SEEK_SET); // Resize the file to the chunk size if needed if ((size_t)size < m_size) { if(ftruncate(m_fd, m_size) < 0) { + // LCOV_EXCL_START YOKAN_LOG_ERROR(MARGO_INSTANCE_NULL, "Failed to resize file %s: %s", m_filename.c_str(), strerror(errno)); close(m_fd); return Status::IOError; + // LCOV_EXCL_STOP } } // Memory-map the file m_data = mmap(nullptr, m_size, PROT_READ | PROT_WRITE, MAP_SHARED, m_fd, 0); if (m_data == MAP_FAILED) { + // LCOV_EXCL_START YOKAN_LOG_ERROR(MARGO_INSTANCE_NULL, "Failed to mmap file %s: %s", m_filename.c_str(), strerror(errno)); close(m_fd); return Status::IOError; + // LCOV_EXCL_STOP } return Status::OK; @@ -627,6 +639,7 @@ class LogDatabase : public DatabaseInterface { static Status create(const std::string& config, DatabaseInterface** kvs) { json cfg; try { + // LCOV_EXCL_START cfg = json::parse(config); if(!cfg.is_object()) return Status::InvalidConf; @@ -642,6 +655,7 @@ class LogDatabase : public DatabaseInterface { return Status::InvalidConf; if(cfg.contains("cache_size") && !cfg["cache_size"].is_number_unsigned()) return Status::InvalidConf; + // LCOV_EXCL_STOP auto chunk_size = cfg.value("chunk_size", 10*1024*1024); cfg["chunk_size"] = chunk_size; diff --git a/tests/python/test-coll-log.py b/tests/python/test-coll-log.py new file mode 100644 index 0000000..05cf4f8 --- /dev/null +++ b/tests/python/test-coll-log.py @@ -0,0 +1,90 @@ +import os +import sys +import unittest +import json +import string +import random + +wd = os.getcwd() +sys.path.append(wd+'/../python') + +from pymargo.core import Engine +import pyyokan_common as yokan +from pyyokan_client import Client +from pyyokan_server import Provider + +class TestUpdate(unittest.TestCase): + + def setUp(self): + self.engine = Engine('tcp') + self.mid = self.engine.get_internal_mid() + self.addr = self.engine.addr() + self.hg_addr = self.addr.get_internal_hg_addr() + self.provider_id = 42 + confg = { + "database": { + "type": "log", + "config": { + "path": "/tmp/py-test-coll-log", + "chunk_size": 16384, + "cache_size": 4 + } + } + } + self.provider = Provider(mid=self.mid, + provider_id=self.provider_id, + config='{"database":{"type":"map"}}') + self.client = Client(mid=self.mid) + self.db = self.client.make_database_handle( + address=self.hg_addr, + provider_id=self.provider_id) + self.coll = self.db.create_collection( + name="matt") + + def tearDown(self): + del self.coll + del self.db + del self.addr + del self.hg_addr + del self.client + del self.mid + del self.provider + self.engine.finalize() + + def test_log(self): + """Test that we can update string documents.""" + reference = list() + letters = string.ascii_letters + # create an store a bunch of documents + # (4096 documents of average size 56 characters = 229376 bytes on average, + # with a chunk size of 16384, we will need 14 chunks on average + for i in range(0, 4096): + doc_len = random.randint(16, 128) + doc = ''.join(random.choice(letters) for i in range(doc_len)) + reference.append(doc) + self.coll.store(doc) + # check (in order) that they have been stored correctly + out_doc = bytearray(128) + for i, doc in enumerate(reference): + doc_len = self.coll.load(id=i, buffer=out_doc) + self.assertEqual(out_doc[0:doc_len].decode("ascii"), doc) + # check some out of order to exercise the cache + for j in range(100): + i = random.randint(0, 4095) + doc_len = self.coll.load(id=i, buffer=out_doc) + self.assertEqual(out_doc[0:doc_len].decode("ascii"), reference[i]) + # do some updates. Some of them are bound to create new chunks, + # some will update in place + for i in range(0, 256): + doc_len = random.randint(16, 128) + doc = ''.join(random.choice(letters) for i in range(doc_len)) + reference[i] = doc + self.coll.update(id=i, document=doc) + # check again all the documents against the reference + for i, doc in enumerate(reference): + doc_len = self.coll.load(id=i, buffer=out_doc) + self.assertEqual(out_doc[0:doc_len].decode("ascii"), doc) + + +if __name__ == '__main__': + unittest.main()