diff --git a/CMakeLists.txt b/CMakeLists.txt index 1479319..ad3e069 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -258,6 +258,11 @@ IF(HAVE_PY_INTERP) ENDIF (NOSETESTS_EXECUTABLE AND CYTHON_FOUND AND HAVE_PY_LIBS) ENDIF(HAVE_PY_INTERP) +FIND_PACKAGE(benchmark CONFIG REQUIRED) +IF (benchmark_FOUND) + MESSAGE(STATUS "Google Benchmark ${benchmark_VERSION} found.") +ENDIF(benchmark_FOUND) + # =================================================================== # Add subdirectories diff --git a/README.md b/README.md index 4995ed0..6373ab9 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,21 @@ centralized here, with the folder names (under that root folder) equal to names of the repositories being benchmarked. As well as likely some common tools placed under some common folder. +## Prerequisites + +### Google Benchmarks + +Google Benchmark is a library supporting C++ benchmarks writing. Use following commands to build and install it: + +``` + git clone https://github.com/google/benchmark.git google-benchmark + mkdir google-benchmark/build + cd google-benchmark/build + cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=OFF .. + make + sudo make install +``` + ## Building Benchmarks Perform the following steps at the shell prompt: @@ -19,4 +34,5 @@ Perform the following steps at the shell prompt: cd build cmake .. make -``` \ No newline at end of file +``` + diff --git a/atomspace/atomspace/CMakeLists.txt b/atomspace/atomspace/CMakeLists.txt index 44e74cb..4354af5 100644 --- a/atomspace/atomspace/CMakeLists.txt +++ b/atomspace/atomspace/CMakeLists.txt @@ -10,6 +10,21 @@ TARGET_LINK_LIBRARIES (atomspace_bm ${COGUTIL_LIBRARY} ) +IF (benchmark_FOUND) + ADD_EXECUTABLE (benchmark + benchmark.cc + scopelink_bm.cc + evaluationlink_bm.cc + ) + + TARGET_LINK_LIBRARIES (benchmark + ${ATOMSPACE_LIBRARIES} + ${COGUTIL_LIBRARY} + benchmark::benchmark + benchmark::benchmark_main + ) +ENDIF(benchmark_FOUND) + IF (HAVE_CYTHON) INCLUDE_DIRECTORIES ( ${PYTHON_INCLUDE_DIRS} diff --git a/atomspace/atomspace/README.md b/atomspace/atomspace/README.md index 1c895d1..9525375 100644 --- a/atomspace/atomspace/README.md +++ b/atomspace/atomspace/README.md @@ -134,3 +134,30 @@ optional arguments: scheme - scheme evaluation functions -i N, --iterations N iterations to average (default=10) ``` + +# Google Benchmark based benchmarks + +Run benchmarks: + +``` +cd /atomspace/atomspace +./benchmark +``` + +Benchmark command line parameters: + +``` +benchmark [--benchmark_list_tests={true|false}] + [--benchmark_filter=] + [--benchmark_min_time=] + [--benchmark_repetitions=] + [--benchmark_report_aggregates_only={true|false} + [--benchmark_format=] + [--benchmark_out=] + [--benchmark_out_format=] + [--benchmark_color={auto|true|false}] + [--benchmark_counters_tabular={true|false}] + [--v=] +``` + +See Google Benchmark documentation https://github.com/google/benchmark/blob/master/README.md and ```DEFINE_*``` definitions in https://github.com/google/benchmark/blob/master/src/benchmark.cc for command line parameters explanation. \ No newline at end of file diff --git a/atomspace/atomspace/benchmark.cc b/atomspace/atomspace/benchmark.cc new file mode 100644 index 0000000..8a5b8c7 --- /dev/null +++ b/atomspace/atomspace/benchmark.cc @@ -0,0 +1,44 @@ +/* + * benchmark.cc + * + * Copyright (C) 2018 OpenCog Foundation + * All Rights Reserved + * + * Created by Vitaly Bogdanov May 29, 2018 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License v3 as + * published by the Free Software Foundation and including the exceptions + * at http://opencog.org/wiki/Licenses + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program; if not, write to: + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include +#include + +#include + +using namespace opencog; + +std::string get_unique_name(const std::string& prefix, size_t& seed) +{ + std::ostringstream oss; + oss << prefix << "-" << ++seed; + return oss.str(); +} + +int main(int argc, char** argv) +{ + logger().set_level(Logger::FINE); + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); +} diff --git a/atomspace/atomspace/benchmark.h b/atomspace/atomspace/benchmark.h new file mode 100644 index 0000000..b9345fd --- /dev/null +++ b/atomspace/atomspace/benchmark.h @@ -0,0 +1,32 @@ +/* + * benchmark.h + * + * Copyright (C) 2018 OpenCog Foundation + * All Rights Reserved + * + * Created by Vitaly Bogdanov May 29, 2018 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License v3 as + * published by the Free Software Foundation and including the exceptions + * at http://opencog.org/wiki/Licenses + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program; if not, write to: + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef ATOMSPACE_ATOMSPACE_BENCHMARK_H_ +#define ATOMSPACE_ATOMSPACE_BENCHMARK_H_ + +#include + +std::string get_unique_name(const std::string& prefix, size_t& seed); + +#endif /* ATOMSPACE_ATOMSPACE_BENCHMARK_H_ */ diff --git a/atomspace/atomspace/evaluationlink_bm.cc b/atomspace/atomspace/evaluationlink_bm.cc new file mode 100644 index 0000000..0686939 --- /dev/null +++ b/atomspace/atomspace/evaluationlink_bm.cc @@ -0,0 +1,106 @@ +/* + * evaluationlink_bm.cc + * + * Copyright (C) 2018 OpenCog Foundation + * All Rights Reserved + * + * Created by Vitaly Bogdanov May 29, 2018 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License v3 as + * published by the Free Software Foundation and including the exceptions + * at http://opencog.org/wiki/Licenses + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program; if not, write to: + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include + +#include +#include +#include +#include +#include + +#include "benchmark.h" + +using namespace opencog; + +static Handle create_evaluation_link(AtomSpace& atomspace, size_t& seed) +{ + Handle X = atomspace.add_node(VARIABLE_NODE, get_unique_name("$X", seed)); + Handle P = atomspace.add_node(PREDICATE_NODE, get_unique_name("P", seed)); + return createLink(EVALUATION_LINK, P, X); +} + +static Handle create_evaluation_link(AtomSpace& atomspace) +{ + size_t seed = 0; + return create_evaluation_link(atomspace, seed); +} + +static void BM_CreateEvaluationLink(benchmark::State& state) +{ + AtomSpace atomspace; + Handle X = atomspace.add_node(VARIABLE_NODE, "$X"); + Handle P = atomspace.add_node(PREDICATE_NODE, "P"); + + for (auto _ : state) + { + createLink(EVALUATION_LINK, P, X); + } +} +BENCHMARK(BM_CreateEvaluationLink); + +static void BM_AddSameEvaluationLink(benchmark::State& state) +{ + AtomSpace atomspace; + Handle evaluationLink = create_evaluation_link(atomspace); + + logger().fine("atomspace size before adding: %d", atomspace.get_size()); + + for (auto _ : state) + { + atomspace.add_atom(evaluationLink); + } + + logger().fine("atomspace size after adding: %d", atomspace.get_size()); + +} +BENCHMARK(BM_AddSameEvaluationLink); + +static void BM_AddEvaluationLink(benchmark::State& state) +{ + AtomSpace atomspace; + + const size_t number_of_links = state.range(0); + std::vector links(number_of_links); + size_t seed = 0; + for (size_t i = 0; i < number_of_links; ++i) + { + links[i] = create_evaluation_link(atomspace, seed); + } + + logger().fine("atomspace size before adding: %d", atomspace.get_size()); + + size_t i = 0; + for (auto _ : state) + { + atomspace.add_atom(links[i++ % number_of_links]); + } + + logger().fine("atomspace size after adding: %d", atomspace.get_size()); +} +BENCHMARK(BM_AddEvaluationLink)->Arg(2<<13)->Arg(2<<14)->Arg(2<<15); + + + + diff --git a/atomspace/atomspace/scopelink_bm.cc b/atomspace/atomspace/scopelink_bm.cc new file mode 100644 index 0000000..9e16da4 --- /dev/null +++ b/atomspace/atomspace/scopelink_bm.cc @@ -0,0 +1,156 @@ +/* + * scopelink_bm.cc + * + * Copyright (C) 2018 OpenCog Foundation + * All Rights Reserved + * + * Created by Vitaly Bogdanov May 28, 2018 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License v3 as + * published by the Free Software Foundation and including the exceptions + * at http://opencog.org/wiki/Licenses + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program; if not, write to: + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "benchmark.h" + +using namespace opencog; + +static Handle create_scope_link(AtomSpace& atomspace, size_t& seed) +{ + Handle X = atomspace.add_node(VARIABLE_NODE, get_unique_name("$X", seed)); + Handle P = atomspace.add_node(PREDICATE_NODE, get_unique_name("P", seed)); + + Handle variable_list = atomspace.add_link(VARIABLE_LIST, X); + Handle expression = atomspace.add_link(EVALUATION_LINK, P, X); + + return createLink(SCOPE_LINK, variable_list, expression); +} + +static Handle create_scope_link(AtomSpace& atomspace) +{ + size_t seed = 0; + return create_scope_link(atomspace, seed); +} + +static void BM_CreateScopeLinkWithVariableList(benchmark::State& state) +{ + AtomSpace atomspace; + + Handle X = atomspace.add_node(VARIABLE_NODE, "$X"); + Handle P = atomspace.add_node(PREDICATE_NODE, "P"); + + Handle variable_list = atomspace.add_link(VARIABLE_LIST, X); + Handle expression = atomspace.add_link(EVALUATION_LINK, P, X); + + Handle h; + for (auto _ : state) + { + h = createLink(SCOPE_LINK, variable_list, expression); + } + + logger().fine( + "variables: " + ScopeLinkCast(h)->get_variables().to_string()); +} +BENCHMARK(BM_CreateScopeLinkWithVariableList); + +static void BM_CreateScopeLinkWithoutVariableList(benchmark::State& state) +{ + AtomSpace atomspace; + + Handle X = atomspace.add_node(VARIABLE_NODE, "$X"); + Handle P = atomspace.add_node(PREDICATE_NODE, "P"); + + Handle expression = atomspace.add_link(EVALUATION_LINK, P, X); + + Handle h; + for (auto _ : state) + { + h = createLink(SCOPE_LINK, expression); + } + + logger().fine( + "variables: " + ScopeLinkCast(h)->get_variables().to_string()); +} +BENCHMARK(BM_CreateScopeLinkWithoutVariableList); + +static void BM_CreateScopeLinkWithLambdaLink(benchmark::State& state) +{ + AtomSpace atomspace; + + Handle X = atomspace.add_node(VARIABLE_NODE, "$X"); + Handle P = atomspace.add_node(PREDICATE_NODE, "P"); + + Handle expression = atomspace.add_link(LAMBDA_LINK, + atomspace.add_link(EVALUATION_LINK, P, X)); + + Handle h; + for (auto _ : state) + { + h = createLink(SCOPE_LINK, expression); + } + + logger().fine( + "variables: " + ScopeLinkCast(h)->get_variables().to_string()); +} +BENCHMARK(BM_CreateScopeLinkWithLambdaLink); + +static void BM_AddSameScopeLink(benchmark::State& state) +{ + AtomSpace atomspace; + Handle scopeLink = create_scope_link(atomspace); + + logger().fine("atomspace size before adding: %d", atomspace.get_size()); + + for (auto _ : state) + { + atomspace.add_atom(scopeLink); + } + + logger().fine("atomspace size after adding: %d", atomspace.get_size()); +} +BENCHMARK(BM_AddSameScopeLink); + +static void BM_AddScopeLink(benchmark::State& state) +{ + AtomSpace atomspace; + + const size_t number_of_links = state.range(0); + std::vector links(number_of_links); + size_t seed = 0; + for (size_t i = 0; i < number_of_links; ++i) + { + links[i] = create_scope_link(atomspace, seed); + } + + logger().fine("atomspace size before adding: %d", atomspace.get_size()); + + size_t i = 0; + for (auto _ : state) + { + atomspace.add_atom(links[i++ % number_of_links]); + } + + logger().fine("atomspace size after adding: %d", atomspace.get_size()); +} +BENCHMARK(BM_AddScopeLink)->Arg(2<<11)->Arg(2<<12)->Arg(2<<13); +