Merge pull request #2 from usdAG/feat/add_new_database_option

Add new database option MariaDB

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "db_creation_src/SQLiteCpp"]
 	path = db_creation_src/SQLiteCpp
 	url = https://github.com/SRombauts/SQLiteCpp.git
+[submodule "db_creation_src/mariadb-connector-cpp"]
+	path = db_creation_src/mariadb-connector-cpp
+	url = https://github.com/mariadb-corporation/mariadb-connector-cpp

README.md

@@ -89,7 +89,6 @@ It is also possible to run a web server that provides this tool's functionality
 ```bash
 ./web_server.py
 ```
-Furthermore, you can use ``gunicorn`` to make the web server more scalable; for example by running:
 ```bash
 gunicorn --worker-class=gevent --worker-connections=50 --workers=3 --bind '0.0.0.0:8000' wsgi:app
 ```
@@ -100,5 +99,22 @@ Finally, you can also use Nginx as a reverse proxy. A sample configuration file
 gunicorn --worker-class=gevent --worker-connections=50 --workers=3 --bind 'unix:/tmp/gunicorn.sock' wsgi:app
 ```
 
+## MariaDB as Second Database Option
+As alternative to the preconfigured SQLite, you can use *MariaDB* as database. A sample configuration file for MariaDB is provided in [``config_mariadb.json``](https://github.com/ra1nb0rn/search_vulns/blob/master/config_mariadb.json).
+
+Make sure that you adjust the values for MariaDB in the configuration file to your MariaDB configuration (*user*, *password*, *host*, *port*).
+
+To use MariaDB instead of *SQLite* for the webserver, simply change the CONFIG_FILE variable in ``web_server.py`` to your config file (e.g. ``config_mariadb.json``).
+It is recommend to change the following values in ``/etc/my.cnf`` to improve the performance of MariaDB:
+```
+[mariadb]
+
+query_cache_type = 1
+query_cache_size = 192M
+innodb_buffer_pool_size = 8G
+thread_handling = pool-of-threads
+```
+`innodb_buffer_pool_size` should be set to approximately 80% of available memory (see [the official documentation](https://mariadb.com/kb/en/innodb-system-variables/#innodb_buffer_pool_size)).
+
 ## License
 *search_vulns* is licensed under the MIT license, see [here](https://github.com/ra1nb0rn/search_vulns/blob/master/LICENSE).

config.json

@@ -1,11 +1,14 @@
 {
-    "DATABASE_FILE": "vulndb.db3",
+    "DATABASE_NAME": "vulndb.db3",
     "MAN_EQUIVALENT_CPES_FILE": "man_equiv_cpes.json",
     "CVE_EDB_MAP_FILE": "cveid_to_edbid.json",
     "NVD_API_KEY": "",
     "cpe_search": {
-        "CPE_DATABASE_FILE": "cpe_search/cpe-search-dictionary.db3",
+        "DATABASE_NAME": "cpe_search/cpe-search-dictionary.db3",
         "DEPRECATED_CPES_FILE": "cpe_search/deprecated-cpes.json",
         "NVD_API_KEY": ""
+    },
+    "DATABASE": {
+        "TYPE": "sqlite"
     }
 }

config_mariadb.json

@@ -0,0 +1,18 @@
+{
+    "DATABASE_NAME": "vulndb",
+    "MAN_EQUIVALENT_CPES_FILE": "man_equiv_cpes.json",
+    "CVE_EDB_MAP_FILE": "cveid_to_edbid.json",
+    "NVD_API_KEY": "",
+    "cpe_search": {
+        "DATABASE_NAME": "cpe_search_dictionary",
+        "DEPRECATED_CPES_FILE": "cpe_search/deprecated-cpes.json",
+        "NVD_API_KEY": ""
+    },
+    "DATABASE": {
+        "TYPE": "mariadb",
+        "HOST": "localhost",
+        "USER": "search_vulns",
+        "PASSWORD": "",
+        "PORT": 3306
+    }
+}

create_sql_statements.json

@@ -0,0 +1,30 @@
+{
+    "TABLES": {
+        "CVE": {
+            "sqlite": "DROP TABLE IF EXISTS cve; CREATE TABLE cve (cve_id VARCHAR(25), description TEXT, edb_ids TEXT, published DATETIME, last_modified DATETIME, cvss_version CHAR(3), base_score CHAR(3), vector VARCHAR(60), severity VARCHAR(15), PRIMARY KEY(cve_id))",
+            "mariadb": "CREATE OR REPLACE TABLE cve (cve_id VARCHAR(25) CHARACTER SET ascii, description TEXT, edb_ids TEXT CHARACTER SET ascii, published DATETIME, last_modified DATETIME, cvss_version CHAR(3) CHARACTER SET ascii, base_score CHAR(4) CHARACTER SET ascii, vector VARCHAR(60) CHARACTER SET ascii, severity VARCHAR(15) CHARACTER SET ascii, PRIMARY KEY(cve_id));"
+        },
+        "CVE_CPE": {
+            "sqlite": "DROP TABLE IF EXISTS cve_cpe; CREATE TABLE cve_cpe (cve_id VARCHAR(25), cpe VARCHAR(255), cpe_version_start VARCHAR(255), is_cpe_version_start_including BOOL, cpe_version_end VARCHAR(255), is_cpe_version_end_including BOOL, PRIMARY KEY(cve_id, cpe, cpe_version_start, is_cpe_version_start_including, cpe_version_end, is_cpe_version_end_including))",
+            "mariadb": "CREATE OR REPLACE TABLE cve_cpe (cve_id VARCHAR(25) CHARACTER SET ascii, cpe VARCHAR(255) CHARACTER SET utf8, cpe_version_start VARCHAR(255)  CHARACTER SET utf8, is_cpe_version_start_including BOOL, cpe_version_end VARCHAR(255)  CHARACTER SET utf8, is_cpe_version_end_including BOOL, PRIMARY KEY(cve_id, cpe, cpe_version_start, is_cpe_version_start_including, cpe_version_end, is_cpe_version_end_including), INDEX(cpe) USING BTREE);"
+        },
+        "CVE_NVD_EXPLOITS_REFS": {
+            "sqlite": "DROP TABLE IF EXISTS cve_nvd_exploits_refs; CREATE TABLE cve_nvd_exploits_refs (cve_id VARCHAR(25), ref_id INTEGER, PRIMARY KEY (cve_id, ref_id))",
+            "mariadb": "CREATE OR REPLACE TABLE cve_nvd_exploits_refs (cve_id VARCHAR(25) CHARACTER SET ascii, ref_id INTEGER, PRIMARY KEY (cve_id, ref_id));"
+        },
+        "CVE_POC_IN_GITHUB_MAP": {
+            "sqlite": "DROP TABLE IF EXISTS cve_poc_in_github_map; CREATE TABLE cve_poc_in_github_map (cve_id VARCHAR(25), reference VARCHAR(255), PRIMARY KEY (cve_id, reference));",
+            "mariadb": "CREATE OR REPLACE TABLE cve_poc_in_github_map (cve_id VARCHAR(25) CHARACTER SET ascii, reference VARCHAR(255), PRIMARY KEY (cve_id, reference));"
+        },
+        "NVD_EXPLOITS_REFS": {
+            "sqlite": "DROP TABLE IF EXISTS nvd_exploits_refs; CREATE TABLE nvd_exploits_refs (ref_id INTEGER, exploit_ref text, PRIMARY KEY (ref_id))",
+            "mariadb": "CREATE OR REPLACE TABLE nvd_exploits_refs (ref_id INTEGER, exploit_ref TEXT CHARACTER SET ascii, PRIMARY KEY (ref_id));"
+        }
+    },
+    "VIEWS" : {
+        "NVD_EXPLOITS_REFS_VIEW": {
+            "sqlite": "DROP VIEW IF EXISTS nvd_exploits_refs_view; CREATE VIEW nvd_exploits_refs_view AS SELECT cve_id, exploit_ref FROM nvd_exploits_refs INNER JOIN cve_nvd_exploits_refs ON nvd_exploits_refs.ref_id = cve_nvd_exploits_refs.ref_id",
+            "mariadb": "CREATE OR REPLACE VIEW nvd_exploits_refs_view AS SELECT cve_id, exploit_ref FROM nvd_exploits_refs INNER JOIN cve_nvd_exploits_refs ON nvd_exploits_refs.ref_id = cve_nvd_exploits_refs.ref_id;"
+        }
+    }
+}

db_creation_src/CMakeLists.txt

@@ -1,20 +1,29 @@
 project(cve-database-creation)
 
-SET(CMAKE_CXX_STANDARD 14)
+SET(CMAKE_CXX_STANDARD 17)
 SET(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 # Include SQLiteCpp library and build it
 option(SQLITECPP_RUN_CPPLINT OFF)
 include_directories(SQLiteCpp/include)
 add_subdirectory(SQLiteCpp)
 
+# Include mariadb-connector-cpp library
+set(CONC_WITH_UNIT_TESTS OFF)
+set(CMAKE_BUILD_TYPE "RelWithDebInfo")
+set(WITH_UNIT_TESTS OFF CACHE INTERNAL "")
+include_directories(mariadb-connector-cpp/include)
+# workaround until mariadb fix issue in test/CMakeLists.txt
+include_directories("${CMAKE_BINARY_DIR}/mariadb-connector-cpp/test")
+add_subdirectory(mariadb-connector-cpp)
+
 # Include Json C++ file
 include_directories(json/single_include)
 
 SET_target_properties(sqlite3 PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
-ADD_EXECUTABLE(create_db create_db.cpp)
-TARGET_LINK_LIBRARIES(create_db SQLiteCpp sqlite3 pthread)
+ADD_EXECUTABLE(create_db create_db.cpp database_wrapper.cpp prepared_statement.cpp)
+TARGET_LINK_LIBRARIES(create_db SQLiteCpp sqlite3 pthread mariadbcpp)
 if (NOT APPLE)
     TARGET_LINK_LIBRARIES(create_db dl)
 endif()