From a2b285f84ae7ab41cfa36dce6dd31b261145ce9f Mon Sep 17 00:00:00 2001 From: Youssef Mellouli Date: Tue, 21 Nov 2023 18:31:28 +0100 Subject: [PATCH] copy database to another instance --- .../checker/src/main/resources/config.json | 7 ++- .../fbs/services/db/DBOperationsService.scala | 7 +-- .../services/db/PsqlOperationsService.scala | 26 +++++++++- .../runner/SQLPlaygroundService.scala | 10 +++- .../services/runner/SQLRunnerService.scala | 5 +- .../runner/SqlPlaygroundVerticle.scala | 52 +++++++++++++++---- 6 files changed, 87 insertions(+), 20 deletions(-) diff --git a/modules/fbs-runner/checker/src/main/resources/config.json b/modules/fbs-runner/checker/src/main/resources/config.json index 6058fab43..d8bd6b00a 100644 --- a/modules/fbs-runner/checker/src/main/resources/config.json +++ b/modules/fbs-runner/checker/src/main/resources/config.json @@ -21,5 +21,10 @@ "SQL_PLAYGROUND_PSQL_SERVER_PASSWORD": "SqfyBWhiFGr7FK60cVR2rel", "SQL_PLAYGROUND_PSQL_SERVER_URL": "jdbc:postgresql://localhost:5432?allowMultiQueries=true", "SQL_QUERY_TIMEOUT_S": 10, - "SQL_MAX_IDLE_TIME": 10 + "SQL_MAX_IDLE_TIME": 10, + "PSQL_SHARING_SERVER_HOST": "psql-sharing", + "PSQL_SHARING_SERVER_PORT": "5444", + "PSQL_SHARING_SERVER_URL": "jdbc:postgresql://psql-sharing:5444", + "PSQL_SHARING_SERVER_USERNAME": "postgres", + "PSQL_SHARING_SERVER_PASSWORD": "R$7pWqY@K5zE3Xt&g9L1MfD" } diff --git a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/DBOperationsService.scala b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/DBOperationsService.scala index a78fd0189..ed549fb11 100644 --- a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/DBOperationsService.scala +++ b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/DBOperationsService.scala @@ -18,10 +18,11 @@ abstract case class DBOperationsService(dbName: String, username: String, queryT def initDB(client: JDBCClient, query: String): Future[ResultSet] = { queryFutureWithTimeout(client, query) } +/** def copyDatabase(targetClient: JDBCClient, targetDBName: String): Future[ResultSet] - - def createUserWithAccess(targetClient: JDBCClient, targetDBName: String): Future[String] - +*/ + def createUserWithAccess(targetClient: JDBCClient, targetDBName: String, targetHost: String): Future[String] + def copyDatabaseToAnotherInstance(sourceHost: String, sourceDBName: String, targetHost: String, targetDBName: String): Future[Unit] def createUserWithWriteAccess(client: JDBCClient, skipUserCreation: Boolean = false): Future[String] def createUserIfNotExist(client: SQLConnection, password: String): Future[ResultSet] diff --git a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/PsqlOperationsService.scala b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/PsqlOperationsService.scala index 6e2e1bf0d..bf452308c 100644 --- a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/PsqlOperationsService.scala +++ b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/PsqlOperationsService.scala @@ -4,9 +4,11 @@ import de.thm.ii.fbs.types.PsqlPrivileges import io.vertx.lang.scala.json.JsonArray import io.vertx.scala.ext.jdbc.JDBCClient import io.vertx.scala.ext.sql.{ResultSet, SQLConnection} +import scala.sys.process._ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future +import scala.sys.process.Process class PsqlOperationsService(override val dbName: String, override val username: String, override val queryTimeout: Int) extends DBOperationsService(dbName, username, queryTimeout) { @@ -152,12 +154,32 @@ class PsqlOperationsService(override val dbName: String, override val username: client.queryFuture(s"$tables $constrains $views $routines $triggers") } +/** override def copyDatabase(targetClient: JDBCClient, targetDBName: String): Future[ResultSet] = { val copyDBQuery = s"CREATE DATABASE $targetDBName WITH TEMPLATE $dbName" targetClient.queryFuture(copyDBQuery) } +*/ +def copyDatabaseToAnotherInstance(sourceHost: String, sourceDBName: String, targetHost: String, targetDBName: String): Future[Unit] = Future { + /**Command to dump the database from the source + * missing case where sourcehost has no password*/ + val dumpCommand = s"pg_dump -h $sourceHost -U $username -F c -b -v -f /tmp/$sourceDBName.dump $sourceDBName" + val dumpExitCode = Process(Seq("bash", "-c", dumpCommand), None, "PGPASSWORD" -> "your_source_db_password").! + + if (dumpExitCode != 0) { + throw new Exception(s"Failed to dump the database from $sourceHost") + } + + // Command to restore the dump to the target + val restoreCommand = s"pg_restore -h $targetHost -U $username -d $targetDBName -v /tmp/$sourceDBName.dump" + val restoreExitCode = Process(Seq("bash", "-c", restoreCommand), None, "PGPASSWORD" -> "your_target_db_password").! + + if (restoreExitCode != 0) { + throw new Exception(s"Failed to restore the database to $targetHost") + } +} - override def createUserWithAccess(targetClient: JDBCClient, targetDBName: String): Future[String] = { + override def createUserWithAccess(targetClient: JDBCClient, targetDBName: String, targetHost: String): Future[String] = { val password = generateUserPassword() val userCreateQuery = s"CREATE USER $username WITH PASSWORD '$password';" val permissionsQuery = s"GRANT ALL PRIVILEGES ON DATABASE $targetDBName TO $username;" @@ -165,6 +187,6 @@ class PsqlOperationsService(override val dbName: String, override val username: for { _ <- targetClient.queryFuture(userCreateQuery) _ <- targetClient.queryFuture(permissionsQuery) - } yield s"postgresql://$username:$password@/$targetDBName" + } yield s"postgresql://$username:$password@$targetHost/$targetDBName" } } diff --git a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/runner/SQLPlaygroundService.scala b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/runner/SQLPlaygroundService.scala index 353d01b9c..251329312 100644 --- a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/runner/SQLPlaygroundService.scala +++ b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/runner/SQLPlaygroundService.scala @@ -6,7 +6,7 @@ import de.thm.ii.fbs.types.{OutputJsonStructure, SqlPlaygroundRunArgs} import de.thm.ii.fbs.util.{DBTypes, DatabaseInformationService, PlaygroundDBConnections, SqlPlaygroundMode} import io.vertx.core.json.JsonObject import io.vertx.scala.ext.sql.ResultSet - +import io.vertx.scala.ext.jdbc.JDBCClient import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future import scala.util.{Failure, Success} @@ -21,6 +21,7 @@ object SQLPlaygroundService { OutputJsonStructure("triggers", Option("manipulation")) ) + def isPlaygroundResult(res: JsonObject): Boolean = res.getString("resultType", "").equals(PLAYGROUND_RESULT_TYPE) /** @@ -70,6 +71,13 @@ class SQLPlaygroundService(val sqlRunArgs: SqlPlaygroundRunArgs, val con: Playgr new PsqlOperationsService(dbName, username, queryTimeout) } + def copyDBAndCreateUser(sourceHost: String, targetHost: String, targetClient: JDBCClient): Future[String] = { + val dbOperations = initDBOperations() + for { + _ <- dbOperations.copyDatabaseToAnotherInstance(sourceHost, dbOperations.dbName, targetHost, dbOperations.dbName) + uri <- dbOperations.createUserWithAccess(targetClient, dbOperations.dbName, targetHost) + } yield uri + } def executeStatement(): Future[(ResultSet, ResultSet)] = { val dbOperations = initDBOperations() val deleteDatabase = SqlPlaygroundMode.shouldDeleteDatabase(sqlRunArgs.mode) diff --git a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/runner/SQLRunnerService.scala b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/runner/SQLRunnerService.scala index 8f5d0e450..93f1ce086 100644 --- a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/runner/SQLRunnerService.scala +++ b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/runner/SQLRunnerService.scala @@ -141,6 +141,7 @@ class SQLRunnerService(val sqlRunArgs: SqlRunArgs, val solutionCon: DBConnection dbOperation } + /** def copyPlaygroundDBAndCreateUser(targetDBHost: String, targetDBPort: Int, dbUser: String, dbPassword: String, originalDBName: String, targetDBName: String): Future[String] = { // Create a client for the target database val targetJDBCClient = JDBCClient.createNonShared(vertx, new JsonObject() @@ -164,7 +165,7 @@ class SQLRunnerService(val sqlRunArgs: SqlRunArgs, val solutionCon: DBConnection s"jdbc:postgresql://$targetDBHost:$targetDBPort/$targetDBName?user=$newUserAndPassword._1&password=$newUserAndPassword._2" } } - +*/ private def buildName(nameExtension: String): String = s"${sqlRunArgs.submissionId}_${sqlRunArgs.runnerId}_$nameExtension" /** @@ -304,7 +305,7 @@ class SQLRunnerService(val sqlRunArgs: SqlRunArgs, val solutionCon: DBConnection (msg, success, correctResults) } -/** def copyPlaygroundDBAndCreateUser(targetDBHost: String, targetDBPort: Int): Future[String] = { + /** def copyPlaygroundDBAndCreateUser(targetDBHost: String, targetDBPort: Int): Future[String] = { val targetDBName = s"${dbName}_copy" val targetJDBCClient = JDBCClient.createNonShared(vertx, new JsonObject() .put("url", s"jdbc:postgresql://$targetDBHost:$targetDBPort/") diff --git a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/verticles/runner/SqlPlaygroundVerticle.scala b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/verticles/runner/SqlPlaygroundVerticle.scala index 807f50034..4c8f8b47a 100644 --- a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/verticles/runner/SqlPlaygroundVerticle.scala +++ b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/verticles/runner/SqlPlaygroundVerticle.scala @@ -1,6 +1,7 @@ package de.thm.ii.fbs.verticles.runner -import de.thm.ii.fbs.services.runner.SQLPlaygroundService +import de.thm.ii.fbs.services.db.PsqlOperationsService +import de.thm.ii.fbs.services.runner.{SQLPlaygroundService} import de.thm.ii.fbs.types._ import de.thm.ii.fbs.util.DBTypes.PSQL_CONFIG_KEY import de.thm.ii.fbs.util.PlaygroundDBConnections @@ -16,27 +17,27 @@ import scala.concurrent.Future import scala.util.{Failure, Success} /** - * Object that stores all static vars for the SqlPlaygroundVerticle - */ + * Object that stores all static vars for the SqlPlaygroundVerticle + */ object SqlPlaygroundVerticle { /** Event Bus Address to start an runner */ val RUN_ADDRESS = "de.thm.ii.fbs.runner.sqlPlayground" } /** - * Verticle that starts the SqlPlayground - * - * @author Max Stephan - */ + * Verticle that starts the SqlPlayground + * + * @author Max Stephan + */ class SqlPlaygroundVerticle extends ScalaVerticle { private val logger = ScalaLogger.getLogger(this.getClass.getName) private var sqlPools = Map[String, SqlPoolWithConfig]() /** - * start SqlRunnerVerticle - * - * @return vertx Future - */ + * start SqlRunnerVerticle + * + * @return vertx Future + */ override def startFuture(): Future[_] = { val psqlDataSource = s"${PSQL_CONFIG_KEY}_playground" val psqlConfig = new JsonObject() @@ -53,6 +54,30 @@ class SqlPlaygroundVerticle extends ScalaVerticle { vertx.eventBus().consumer(RUN_ADDRESS, startSqlPlayground).completionFuture() } + private def copyDatabaseAndCreateUser(runArgs: SqlPlaygroundRunArgs): Future[String] = { + // Extract sqlRunArgs from the message + + + // Get the PlaygroundDBConnections + val con = getConnection(runArgs).getOrElse(throw new Exception("Failed to get database connection")) + + // Get the query timeout from config + val queryTimeout = config.getInteger("SQL_QUERY_TIMEOUT_S", 10) + + // Create an instance of SQLPlaygroundService + val sqlPlaygroundService = new SQLPlaygroundService(runArgs, con, queryTimeout) + + // Extract source and target host + val sourceHost = config.getString("PSQL_SERVER_URL", "jdbc:postgresql://localhost:5432").split("//")(1).split("/")(0) + val targetHost = config.getString("PSQL_SHARING_SERVER_URL", "jdbc:postgresql://psql-sharing:5432").split("//")(1).split("/")(0) + + // Get the JDBC client for the target database + val targetClient = sqlPools.getOrElse(PSQL_CONFIG_KEY, throw new Exception("PSQL config not found")).pool + + // Call the method to copy database and create user + sqlPlaygroundService.copyDBAndCreateUser(sourceHost, targetHost, targetClient) + } + private def startSqlPlayground(msg: Message[JsonObject]): Future[Unit] = Future { val runArgs = msg.body().mapTo(classOf[SqlPlaygroundRunArgs]) @@ -63,6 +88,11 @@ class SqlPlaygroundVerticle extends ScalaVerticle { if (con.isDefined) { executeQueries(runArgs, con.get) + // Call the method to copy database and create user + copyDatabaseAndCreateUser(runArgs).onComplete { + case Success(uri) => logger.info(s"Database copied, new URI: $uri") + case Failure(ex) => logger.error("Error in copying database and creating user", ex) + } } } catch { case e: Throwable => handleError(runArgs, e)