From 3f943ab475e0f6dbb2741a182a47ed7e3e5ee8a5 Mon Sep 17 00:00:00 2001 From: Jonas Kuche Date: Sun, 12 Nov 2023 21:49:46 +0100 Subject: [PATCH] feat(playground): change options and add metrics --- modules/fbs-runner/checker/build.gradle | 7 +++++++ .../services/db/PsqlOperationsService.scala | 16 ++++++++++++++ .../scala/de/thm/ii/fbs/util/Metrics.scala | 21 +++++++++++++++++++ .../runner/SqlPlaygroundVerticle.scala | 18 ++++++++++++---- 4 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/util/Metrics.scala diff --git a/modules/fbs-runner/checker/build.gradle b/modules/fbs-runner/checker/build.gradle index 4639de68a..c07a53829 100644 --- a/modules/fbs-runner/checker/build.gradle +++ b/modules/fbs-runner/checker/build.gradle @@ -35,6 +35,13 @@ dependencies { implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.14.2' implementation 'mysql:mysql-connector-java:8.0.32' implementation 'org.postgresql:postgresql:42.5.3' + implementation platform("io.opentelemetry:opentelemetry-bom:1.31.0") + implementation platform('io.opentelemetry:opentelemetry-bom-alpha:1.31.0-alpha') + implementation("io.opentelemetry:opentelemetry-api"); + implementation("io.opentelemetry:opentelemetry-sdk"); + implementation("io.opentelemetry:opentelemetry-sdk-metrics"); + implementation("io.opentelemetry:opentelemetry-exporter-logging"); + implementation('io.opentelemetry:opentelemetry-exporter-prometheus') testImplementation 'junit:junit:4.13.2' } diff --git a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/PsqlOperationsService.scala b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/PsqlOperationsService.scala index 12d66c896..8274980b9 100644 --- a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/PsqlOperationsService.scala +++ b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/services/db/PsqlOperationsService.scala @@ -7,6 +7,7 @@ import io.vertx.scala.ext.sql.{ResultSet, SQLConnection} import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future +import scala.util.{Failure, Success, Try} class PsqlOperationsService(override val dbName: String, override val username: String, override val queryTimeout: Int) extends DBOperationsService(dbName, username, queryTimeout) { @@ -152,4 +153,19 @@ class PsqlOperationsService(override val dbName: String, override val username: client.queryFuture(s"$tables $constrains $views $routines $triggers") } + + override def queryFutureWithTimeout(client: JDBCClient, sql: String): Future[ResultSet] = { + client.getConnectionFuture().flatMap(con => { + con.queryFuture(s"SET statement_timeout = ${queryTimeout*1000};").flatMap(_ => { + con.queryFuture(sql) transform { + case Success(result) => + con.close() + Try(result) + case Failure(exception) => + con.close() + Failure(throw exception) + } + }) + }) + } } diff --git a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/util/Metrics.scala b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/util/Metrics.scala new file mode 100644 index 000000000..f3565d093 --- /dev/null +++ b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/util/Metrics.scala @@ -0,0 +1,21 @@ +package de.thm.ii.fbs.util + +import io.opentelemetry.exporter.prometheus.PrometheusHttpServer +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.resources.Resource + + +object Metrics { + private val prometheusHttpServer = PrometheusHttpServer.builder().build() + val openTelemetry: OpenTelemetrySdk = initOpenTelemetry() + + private def initOpenTelemetry(): OpenTelemetrySdk = { + // Include required service.name resource attribute on all spans and metrics + val resource = Resource.getDefault.merge(Resource.builder.put("service-name", "fbs-runner").build) + val openTelemetrySdk = OpenTelemetrySdk.builder.setMeterProvider( + SdkMeterProvider.builder.setResource(resource).registerMetricReader(prometheusHttpServer).build + ).buildAndRegisterGlobal + openTelemetrySdk + } +} diff --git a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/verticles/runner/SqlPlaygroundVerticle.scala b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/verticles/runner/SqlPlaygroundVerticle.scala index 807f50034..65fcf8901 100644 --- a/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/verticles/runner/SqlPlaygroundVerticle.scala +++ b/modules/fbs-runner/checker/src/main/scala/de/thm/ii/fbs/verticles/runner/SqlPlaygroundVerticle.scala @@ -3,7 +3,7 @@ package de.thm.ii.fbs.verticles.runner import de.thm.ii.fbs.services.runner.SQLPlaygroundService import de.thm.ii.fbs.types._ import de.thm.ii.fbs.util.DBTypes.PSQL_CONFIG_KEY -import de.thm.ii.fbs.util.PlaygroundDBConnections +import de.thm.ii.fbs.util.{Metrics, PlaygroundDBConnections} import de.thm.ii.fbs.verticles.HttpVerticle import de.thm.ii.fbs.verticles.runner.SqlPlaygroundVerticle.RUN_ADDRESS import io.vertx.core.json.JsonObject @@ -12,6 +12,7 @@ import io.vertx.scala.core.eventbus.Message import io.vertx.scala.ext.jdbc.JDBCClient import java.sql.{SQLException, SQLTimeoutException} +import java.util.Date import scala.concurrent.Future import scala.util.{Failure, Success} @@ -31,6 +32,9 @@ object SqlPlaygroundVerticle { class SqlPlaygroundVerticle extends ScalaVerticle { private val logger = ScalaLogger.getLogger(this.getClass.getName) private var sqlPools = Map[String, SqlPoolWithConfig]() + private val meter = Metrics.openTelemetry.meterBuilder("de.thm.mni.ii.fbs.verticles.runner.playground").build() + private val processingCounter = meter.upDownCounterBuilder("processingCount").setDescription("Processing Requests").build() + private val processingTimeCounter = meter.histogramBuilder("processingTime").ofLongs().setDescription("Time for processing").build() /** * start SqlRunnerVerticle @@ -43,9 +47,9 @@ class SqlPlaygroundVerticle extends ScalaVerticle { .put("user", config.getString("SQL_PLAYGROUND_PSQL_SERVER_USERNAME", "root")) .put("password", config.getString("SQL_PLAYGROUND_PSQL_SERVER_PASSWORD", "")) .put("url", config.getString("SQL_PLAYGROUND_PSQL_SERVER_URL", "jdbc:postgresql://localhost:5432")) - .put("max_pool_size", config.getInteger("SQL_PLAYGROUND_INSTANCES", 15)) + .put("max_pool_size", config.getInteger("SQL_PLAYGROUND_INSTANCES", 256)) .put("driver_class", "org.postgresql.Driver") - .put("max_idle_time", config.getInteger("SQL_MAX_IDLE_TIME", 10)) + .put("max_idle_time", config.getInteger("SQL_MAX_IDLE_TIME", 60)) .put("dataSourceName", psqlDataSource) val psqlPool = JDBCClient.createShared(vertx, psqlConfig, psqlDataSource) sqlPools += (PSQL_CONFIG_KEY -> SqlPoolWithConfig(psqlPool, psqlConfig)) @@ -56,6 +60,8 @@ class SqlPlaygroundVerticle extends ScalaVerticle { private def startSqlPlayground(msg: Message[JsonObject]): Future[Unit] = Future { val runArgs = msg.body().mapTo(classOf[SqlPlaygroundRunArgs]) + processingCounter.add(1) + val startTime = new Date().getTime try { logger.info(s"SqlPlayground received execution ${runArgs.executionId}") @@ -66,6 +72,10 @@ class SqlPlaygroundVerticle extends ScalaVerticle { } } catch { case e: Throwable => handleError(runArgs, e) + } finally { + val endTime = new Date().getTime + processingCounter.add(-1) + processingTimeCounter.record(endTime - startTime) } } @@ -81,7 +91,7 @@ class SqlPlaygroundVerticle extends ScalaVerticle { } private def executeQueries(runArgs: SqlPlaygroundRunArgs, con: PlaygroundDBConnections): Unit = { - val sqlPlayground = new SQLPlaygroundService(runArgs, con, config.getInteger("SQL_QUERY_TIMEOUT_S", 10)) + val sqlPlayground = new SQLPlaygroundService(runArgs, con, config.getInteger("SQL_QUERY_TIMEOUT_S", 1)) sqlPlayground.executeStatement().onComplete({ case Success(value) =>