diff --git a/all-in-one/Dockerfile b/all-in-one/Dockerfile index 8e7a2fb..e320a64 100644 --- a/all-in-one/Dockerfile +++ b/all-in-one/Dockerfile @@ -3,13 +3,21 @@ ARG BASE_VERSION=2022-10-27T19-02-22 ARG CORE_VERSION=2.0.4 ARG CONSOLE_VERSION=1.4.6 ARG APISERVER_VERSION=0.0.16 +ARG PROMETHEUS_VERSION=v2.40.7 +ARG PROMTAIL_VERSION=2.9.4 +ARG LOKI_VERSION=2.9.4 +ARG GRAFANA_VERSION=9.3.6 -FROM ${HUB}/api-server:${APISERVER_VERSION} as apiserver -FROM ${HUB}/higress:${CORE_VERSION} as controller -FROM ${HUB}/pilot:${CORE_VERSION} as pilot -FROM ${HUB}/gateway:${CORE_VERSION} as gateway -FROM ${HUB}/console:${CONSOLE_VERSION} as console -FROM eclipse-temurin:21-jre as jdk +FROM ${HUB}/api-server:${APISERVER_VERSION} AS apiserver +FROM ${HUB}/higress:${CORE_VERSION} AS controller +FROM ${HUB}/pilot:${CORE_VERSION} AS pilot +FROM ${HUB}/gateway:${CORE_VERSION} AS gateway +FROM ${HUB}/console:${CONSOLE_VERSION} AS console +FROM ${HUB}/prometheus:${PROMETHEUS_VERSION} AS prometheus +FROM ${HUB}/promtail:${PROMTAIL_VERSION} AS promtail +FROM ${HUB}/loki:${LOKI_VERSION} AS loki +FROM ${HUB}/grafana:${GRAFANA_VERSION} AS grafana +FROM ${HUB}/eclipse-temurin:21-jre AS jdk FROM ${HUB}/base:${BASE_VERSION} @@ -21,12 +29,17 @@ COPY --from=controller /usr/local/bin/higress /usr/local/bin/higress # Install pilot COPY --from=pilot /usr/local/bin/pilot-discovery /usr/local/bin/pilot-discovery -COPY --from=pilot /var/lib/istio/envoy/*.json /var/lib/istio/envoy/ COPY --from=pilot /usr/local/bin/higress-pilot-start.sh /usr/local/bin/higress-pilot-start.sh # Install gateway +COPY --from=gateway /var/lib/istio/envoy/*.json /var/lib/istio/envoy/ COPY --from=gateway /usr/local/bin/pilot-agent /usr/local/bin/pilot-agent COPY --from=gateway /usr/local/bin/envoy /usr/local/bin/envoy +COPY --from=gateway /usr/local/bin/higress-proxy-*.sh /usr/local/bin/ +RUN chmod a+x /usr/local/bin/higress-proxy-container-init.sh; \ + sed -i 's/1337/0/g' /usr/local/bin/higress-proxy-container-init.sh; \ + /usr/local/bin/higress-proxy-container-init.sh +COPY --from=gateway /usr/local/bin/supercronic* /usr/local/bin/ # Install console COPY --from=console /app /app @@ -36,16 +49,28 @@ ENV JAVA_HOME=/opt/java/openjdk COPY --from=jdk $JAVA_HOME $JAVA_HOME ENV PATH="${JAVA_HOME}/bin:${PATH}" -# Install supervisord and initialize related folders -RUN apt-get update && \ - apt-get install --no-install-recommends -y \ - supervisor \ +# Install Prometheus +COPY --from=prometheus /bin/prometheus /usr/local/bin/prometheus + +# Install Promtail +COPY --from=promtail /usr/bin/promtail /usr/local/bin/promtail + +# Install Loki +COPY --from=loki /usr/bin/loki /usr/local/bin/loki + +# Install Grafana +COPY --from=grafana /usr/share/grafana /usr/share/grafana +COPY --from=grafana /run.sh /usr/local/bin/grafana.sh + +# Install supervisord, logrotate, cron and initialize related folders +RUN apt-get update --allow-unauthenticated && \ + apt-get install --no-install-recommends -y --allow-unauthenticated \ + supervisor logrotate cron \ + && apt-get upgrade -y --allow-unauthenticated \ && apt-get clean \ && rm -rf /var/log/*log /var/lib/apt/lists/* /var/log/apt/* /var/lib/dpkg/*-old /var/cache/debconf/*-old \ && mkdir -p /var/log/higress \ - && chown 1337:1337 -R /var/log/higress \ - && mkdir /data \ - && chown 1337:1337 -R /data + && mkdir /data COPY ./supervisord/supervisord.conf /etc/supervisor/conf.d/supervisord.conf # Initialize configurations @@ -54,6 +79,10 @@ COPY ./gateway/podinfo /etc/istio/pod COPY ./scripts /usr/local/bin COPY ./apiserver/config /app/kubeconfig COPY ./config /opt/data/defaultConfig +COPY ./prometheus /etc/prometheus +COPY ./promtail /etc/promtail +COPY ./loki /etc/loki +COPY ./grafana /etc/grafana EXPOSE 8080 8443 8001 diff --git a/all-in-one/config/configmaps/higress-config.yaml b/all-in-one/config/configmaps/higress-config.yaml new file mode 100644 index 0000000..bb87e9f --- /dev/null +++ b/all-in-one/config/configmaps/higress-config.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: higress-config + namespace: higress-system + creationTimestamp: "2000-01-01T00:00:00Z" + resourceVersion: "1" +data: + higress: |- + downstream: + connectionBufferLimits: 32768 + http2: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65535 + maxConcurrentStreams: 100 + idleTimeout: 180 + maxRequestHeadersKb: 60 + routeTimeout: 0 + upstream: + connectionBufferLimits: 10485760 + idleTimeout: 10 \ No newline at end of file diff --git a/all-in-one/get-ai-gateway.sh b/all-in-one/get-ai-gateway.sh index b602a04..d65d5f7 100755 --- a/all-in-one/get-ai-gateway.sh +++ b/all-in-one/get-ai-gateway.sh @@ -156,7 +156,7 @@ runConfigWizard() { "Zhipu AI|ZHIPUAI" "Ollama|OLLAMA|configureOllamaProvider" "Claude|CLAUDE|configureClaudeProvider" - "Baidu AI Cloud|BAIDU" + # "Baidu AI Cloud|BAIDU" # "Tencent Hunyuan|HUNYUAN" "Stepfun|STEPFUN" "Minimax|MINIMAX|configureMinimaxProvider" @@ -324,6 +324,7 @@ ${env}=${!env}" done cat <$DATA_FOLDER/$CONFIG_FILENAME MODE=full +o11y=on CONFIG_TEMPLATE=ai-gateway GATEWAY_HTTP_PORT=${GATEWAY_HTTP_PORT} GATEWAY_HTTPS_PORT=${GATEWAY_HTTPS_PORT} diff --git a/all-in-one/grafana/grafana.ini b/all-in-one/grafana/grafana.ini new file mode 100644 index 0000000..218ee95 --- /dev/null +++ b/all-in-one/grafana/grafana.ini @@ -0,0 +1,21 @@ +[server] +protocol=http +domain=localhost +root_url="%(protocol)s://%(domain)s/grafana" +serve_from_sub_path=true + +[auth] +disable_login_form=true +disable_signout_menu=true + +[auth.anonymous] +enabled=true +org_name=Main Org. +org_role=Viewer + +[users] +default_theme=light +viewers_can_edit=true + +[security] +allow_embedding=true \ No newline at end of file diff --git a/all-in-one/loki/config.yaml b/all-in-one/loki/config.yaml new file mode 100644 index 0000000..1b2217c --- /dev/null +++ b/all-in-one/loki/config.yaml @@ -0,0 +1,49 @@ +auth_enabled: false +common: + compactor_address: 'loki' + path_prefix: /var/loki + replication_factor: 1 + storage: + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules +frontend: + scheduler_address: "" +frontend_worker: + scheduler_address: "" +index_gateway: + mode: simple +limits_config: + max_cache_freshness_per_query: 10m + reject_old_samples: true + reject_old_samples_max_age: 168h + split_queries_by_interval: 15m +memberlist: + join_members: + - localhost +query_range: + align_queries_with_step: true +ruler: + storage: + type: local +runtime_config: + file: /etc/loki/runtime-config.yaml +schema_config: + configs: + - from: "2022-01-11" + index: + period: 24h + prefix: loki_index_ + object_store: filesystem + schema: v12 + store: boltdb-shipper +server: + http_listen_port: 3100 + grpc_listen_port: 9095 +storage_config: + hedging: + at: 250ms + max_per_second: 20 + up_to: 3 +tracing: + enabled: false \ No newline at end of file diff --git a/all-in-one/loki/runtime-config.yaml b/all-in-one/loki/runtime-config.yaml new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/all-in-one/loki/runtime-config.yaml @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/all-in-one/meshConfig/mesh b/all-in-one/meshConfig/mesh index 4ed3d01..7527eb1 100644 --- a/all-in-one/meshConfig/mesh +++ b/all-in-one/meshConfig/mesh @@ -16,6 +16,9 @@ dnsRefreshRate: 200s enableAutoMtls: false enablePrometheusMerge: true ingressControllerMode: "OFF" +mseIngressGlobalConfig: + enableH3: false + enableProxyProtocol: false protocolDetectionTimeout: 100ms rootNamespace: higress-system trustDomain: cluster.local diff --git a/all-in-one/prometheus/prometheus.yaml b/all-in-one/prometheus/prometheus.yaml new file mode 100644 index 0000000..abb24e1 --- /dev/null +++ b/all-in-one/prometheus/prometheus.yaml @@ -0,0 +1,17 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s +scrape_configs: + - job_name: 'prometheus' + metrics_path: /prometheus/metrics + static_configs: + - targets: ['localhost:9090'] + - job_name: 'gateway' + metrics_path: /stats/prometheus + static_configs: + - targets: ['localhost:15020'] + labels: + container: 'higress-gateway' + namespace: 'higress-system' + higress: 'higress-system-higress-gateway' + pod: 'higress' \ No newline at end of file diff --git a/all-in-one/promtail/promtail.yaml b/all-in-one/promtail/promtail.yaml new file mode 100644 index 0000000..3c1d534 --- /dev/null +++ b/all-in-one/promtail/promtail.yaml @@ -0,0 +1,54 @@ +server: + log_level: info + http_listen_port: 3101 + grpc_listen_port: 9096 + +clients: +- url: http://localhost:3100/loki/api/v1/push + +positions: + filename: /var/promtail/promtail-positions.yaml +target_config: + sync_period: 10s +scrape_configs: +- job_name: access-logs + static_configs: + - targets: + - localhost + labels: + __path__: /var/log/proxy/access.log + pipeline_stages: + - json: + expressions: + authority: + method: + path: + protocol: + request_id: + response_code: + response_flags: + route_name: + trace_id: + upstream_cluster: + upstream_host: + upstream_transport_failure_reason: + user_agent: + x_forwarded_for: + - labels: + authority: + method: + path: + protocol: + request_id: + response_code: + response_flags: + route_name: + trace_id: + upstream_cluster: + upstream_host: + upstream_transport_failure_reason: + user_agent: + x_forwarded_for: + - timestamp: + source: timestamp + format: RFC3339Nano \ No newline at end of file diff --git a/all-in-one/scripts/base.sh b/all-in-one/scripts/base.sh index 27de79b..6df3f9b 100644 --- a/all-in-one/scripts/base.sh +++ b/all-in-one/scripts/base.sh @@ -27,7 +27,6 @@ function readinessCheck() { function createDir() { sudo mkdir -p "$1" - sudo chown 1337:1337 "$1" } case $MODE in @@ -39,9 +38,19 @@ case $MODE in MODE=full ;; esac - echo "Mode=$MODE" +case $O11Y in + true|TRUE|on|ON|yes|YES) + O11Y=on + ;; + *) + # Default to full mode + O11Y=off + ;; +esac +echo "O11Y=$O11Y" + CONSOLE_USED_MARKER='/data/.console-used' CONSOLE_USED='false' if [ -f "$CONSOLE_USED_MARKER" ]; then diff --git a/all-in-one/scripts/config-template/ai-gateway.sh b/all-in-one/scripts/config-template/ai-gateway.sh index e5b019b..37f5308 100644 --- a/all-in-one/scripts/config-template/ai-gateway.sh +++ b/all-in-one/scripts/config-template/ai-gateway.sh @@ -21,7 +21,7 @@ function initializeLlmProviderConfigs() { initializeLlmProviderConfig yi yi YI api.lingyiwanwu.com initializeLlmProviderConfig deepseek deepseek DEEPSEEK api.deepseek.com initializeLlmProviderConfig zhipuai zhipuai ZHIPUAI open.bigmodel.cn - initializeLlmProviderConfig baidu baidu BAIDU aip.baidubce.com + # initializeLlmProviderConfig baidu baidu BAIDU aip.baidubce.com # initializeLlmProviderConfig hunyuan hunyuan HUNYUAN hunyuan.tencentcloudapi.com 443 "https" "" "${EXTRA_CONFIGS[@]}" initializeLlmProviderConfig stepfun stepfun STEPFUN api.stepfun.com # initializeLlmProviderConfig cloudflare cloudflare CLOUDFLARE api.cloudflare.com 443 "https" "" "${EXTRA_CONFIGS[@]}" diff --git a/all-in-one/scripts/start-console.sh b/all-in-one/scripts/start-console.sh index 6d346fb..a8da295 100644 --- a/all-in-one/scripts/start-console.sh +++ b/all-in-one/scripts/start-console.sh @@ -24,5 +24,12 @@ touch "$CONSOLE_USED_MARKER" set -e -HIGRESS_CONSOLE_KUBE_CONFIG="/app/kubeconfig" SERVER_PORT="$CONSOLE_PORT" \ +if [ "$O11Y" == "on" ]; then + export HIGRESS_CONSOLE_DASHBOARD_BASE_URL="http://localhost:3000/grafana" + export HIGRESS_CONSOLE_DASHBOARD_DATASOURCE_PROM_URL="http://localhost:9090/prometheus" + export HIGRESS_CONSOLE_DASHBOARD_DATASOURCE_LOKI_URL="http://localhost:3100" +fi + +HIGRESS_CONSOLE_KUBE_CONFIG="/app/kubeconfig" \ + SERVER_PORT="$CONSOLE_PORT" \ bash /app/start.sh \ No newline at end of file diff --git a/all-in-one/scripts/start-gateway.sh b/all-in-one/scripts/start-gateway.sh index 9e336a5..de7a95f 100644 --- a/all-in-one/scripts/start-gateway.sh +++ b/all-in-one/scripts/start-gateway.sh @@ -17,8 +17,16 @@ set -e createDir /etc/istio/proxy createDir /var/lib/istio/data +createDir /var/log/proxy +touch /var/log/proxy/access.log -/usr/local/bin/pilot-agent proxy router \ +if [ "$O11Y" == "on" ]; then + sed -i -E 's/^accessLogFile: .+$/accessLogFile: \/var\/log\/proxy\/access.log/' /etc/istio/config/mesh +else + sed -i -E 's/^accessLogFile: .+$/accessLogFile: \/dev\/stdout/' /etc/istio/config/mesh +fi + +/usr/local/bin/higress-proxy-start.sh proxy router \ --domain=higress-system.svc.cluster.local \ --proxyLogLevel=${GATEWAY_LOG_LEVEL:-warning} \ --proxyComponentLogLevel=${GATEWAY_COMPONENT_LOG_LEVEL:-misc:error} \ diff --git a/all-in-one/scripts/start-grafana.sh b/all-in-one/scripts/start-grafana.sh new file mode 100644 index 0000000..a65eea2 --- /dev/null +++ b/all-in-one/scripts/start-grafana.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +cd "$(dirname -- "$0")" +ROOT=$(pwd) +cd - >/dev/null +source $ROOT/base.sh + +if [ "$O11Y" != "on" ]; then + echo "Grafana won't run when o11y is not turned on." + sleep 2 + exit 0 +fi + +set -e + +createDir /var/lib/grafana +createDir /var/log/grafana + +GF_PATHS_CONFIG="/etc/grafana/grafana.ini" \ + GF_PATHS_DATA="/var/lib/grafana" \ + GF_PATHS_HOME="/usr/share/grafana" \ + GF_PATHS_LOGS="/var/log/grafana" \ + GF_PATHS_PLUGINS="/var/lib/grafana/plugins" \ + GF_PATHS_PROVISIONING="/etc/grafana/provisioning" \ + PATH="/usr/share/grafana/bin:$PATH" \ + bash /usr/local/bin/grafana.sh \ No newline at end of file diff --git a/all-in-one/scripts/start-loki.sh b/all-in-one/scripts/start-loki.sh new file mode 100644 index 0000000..3f953dc --- /dev/null +++ b/all-in-one/scripts/start-loki.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +cd "$(dirname -- "$0")" +ROOT=$(pwd) +cd - >/dev/null +source $ROOT/base.sh + +if [ "$O11Y" != "on" ]; then + echo "Loki won't run when o11y is not turned on." + sleep 2 + exit 0 +fi + +set -e + +createDir /var/loki/chunks +createDir /var/loki/rules + +/usr/local/bin/loki \ + -config.file=/etc/loki/config.yaml \ + -target=all \ No newline at end of file diff --git a/all-in-one/scripts/start-pilot.sh b/all-in-one/scripts/start-pilot.sh index c9903b1..8d17a17 100644 --- a/all-in-one/scripts/start-pilot.sh +++ b/all-in-one/scripts/start-pilot.sh @@ -95,4 +95,6 @@ initCerts --log_output_level=default:info \ --domain=cluster.local \ --keepaliveMaxServerConnectionAge=30m \ - --caCertFile=/etc/certs/ca-cert.pem \ No newline at end of file + --caCertFile=/etc/certs/ca-cert.pem \ + --meshConfig=/etc/istio/config/mesh \ + --networksConfig=/etc/istio/config/meshNetworks \ No newline at end of file diff --git a/all-in-one/scripts/start-prometheus.sh b/all-in-one/scripts/start-prometheus.sh new file mode 100644 index 0000000..19e51df --- /dev/null +++ b/all-in-one/scripts/start-prometheus.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +cd "$(dirname -- "$0")" +ROOT=$(pwd) +cd - >/dev/null +source $ROOT/base.sh + +if [ "$O11Y" != "on" ]; then + echo "Prometheus won't run when o11y is not turned on." + sleep 2 + exit 0 +fi + +set -e + +createDir /prometheus + +/usr/local/bin/prometheus \ + --config.file=/etc/prometheus/prometheus.yaml \ + --web.external-url=/prometheus \ + --storage.tsdb.path=/prometheus \ + --storage.tsdb.retention=6h \ No newline at end of file diff --git a/all-in-one/scripts/start-promtail.sh b/all-in-one/scripts/start-promtail.sh new file mode 100644 index 0000000..f990fa1 --- /dev/null +++ b/all-in-one/scripts/start-promtail.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +cd "$(dirname -- "$0")" +ROOT=$(pwd) +cd - >/dev/null +source $ROOT/base.sh + +if [ "$O11Y" != "on" ]; then + echo "Promtail won't run when o11y is not turned on." + sleep 2 + exit 0 +fi + +set -e + +createDir /var/promtail + +/usr/local/bin/promtail -config.file=/etc/promtail/promtail.yaml \ No newline at end of file diff --git a/all-in-one/supervisord/supervisord.conf b/all-in-one/supervisord/supervisord.conf index 359072f..4a6858f 100644 --- a/all-in-one/supervisord/supervisord.conf +++ b/all-in-one/supervisord/supervisord.conf @@ -7,6 +7,7 @@ loglevel=info pidfile=/var/log/higress/supervisord.pid [program:apiserver] +directory=/ command=bash /usr/local/bin/start-apiserver.sh priority=0 autostart=true @@ -18,6 +19,7 @@ redirect_stderr=true environment= [program:controller] +directory=/ command=bash /usr/local/bin/start-controller.sh priority=100 autostart=true @@ -26,9 +28,10 @@ startsecs=1 stdout_logfile=/var/log/higress/controller.log stdout_logfile_maxbytes=10MB redirect_stderr=true -environment=CONTROLLER_KEEP_XDS_CONFIG_LABELS="false",CONTROLLER_KEEP_XDS_CONFIG_ANNOTATIONS="false" +environment=CONTROLLER_KEEP_XDS_CONFIG_LABELS="false",CONTROLLER_KEEP_XDS_CONFIG_ANNOTATIONS="false",PILOT_ENABLE_GATEWAY_API="false",PILOT_ENABLE_ALPHA_GATEWAY_API="false" [program:pilot] +directory=/ command=bash /usr/local/bin/start-pilot.sh priority=200 autostart=true @@ -37,9 +40,10 @@ startsecs=1 stdout_logfile=/var/log/higress/pilot.log stdout_logfile_maxbytes=10MB redirect_stderr=true -environment=REVISION="default",JWT_POLICY="none",POD_NAME="higress-pilot",POD_NAMESPACE="higress-system",ROOT_CA_DIR="/etc/certs",PILOT_CERT_PROVIDER="istiod",KUBECONFIG="/app/kubeconfig",PILOT_ENABLE_CROSS_CLUSTER_WORKLOAD_ENTRY="false",PILOT_ENABLE_METADATA_EXCHANGE="false",PILOT_SCOPE_GATEWAY_TO_NAMESPACE="true",VALIDATION_ENABLED="false",PILOT_TRACE_SAMPLING="1",PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND="true",PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND="true",PILOT_ENABLE_ANALYSIS="false",PILOT_ENABLE_GATEWAY_API="true",PILOT_ENABLE_GATEWAY_API_STATUS="false",PILOT_ENABLE_GATEWAY_API_DEPLOYMENT_CONTROLLER="false",CLUSTER_ID="Kubernetes",CUSTOM_CA_CERT_NAME="higress-ca-root-cert",DEBUG_AUTH="false" +environment=REVISION="default",JWT_POLICY="none",POD_NAME="higress-pilot",POD_NAMESPACE="higress-system",ROOT_CA_DIR="/etc/certs",PILOT_CERT_PROVIDER="istiod",KUBECONFIG="/app/kubeconfig",PILOT_ENABLE_CROSS_CLUSTER_WORKLOAD_ENTRY="false",PILOT_ENABLE_METADATA_EXCHANGE="false",PILOT_SCOPE_GATEWAY_TO_NAMESPACE="true",VALIDATION_ENABLED="false",PILOT_TRACE_SAMPLING="1",PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND="true",PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND="true",PILOT_ENABLE_ANALYSIS="false",PILOT_ENABLE_GATEWAY_API="false",PILOT_ENABLE_ALPHA_GATEWAY_API="false",PILOT_ENABLE_GATEWAY_API_STATUS="false",PILOT_ENABLE_GATEWAY_API_DEPLOYMENT_CONTROLLER="false",PILOT_ENABLE_ALPN_FILTER="true",VALIDATION_WEBHOOK_CONFIG_NAME="",ISTIO_DUAL_STACK="false",ENABLE_OPTIMIZED_CONFIG_REBUILD="false",PILOT_ENABLE_K8S_SELECT_WORKLOAD_ENTRIES="false",DEFAULT_UPSTREAM_CONCURRENCY_THRESHOLD="10000",ISTIO_GPRC_MAXRECVMSGSIZE="104857600",ENBALE_SCOPED_RDS="true",ON_DEMAND_RDS="false",HOST_RDS_MERGE_SUBSET="false",PILOT_FILTER_GATEWAY_CLUSTER_CONFIG="true",PRIORITIZED_LEADER_ELECTION="false",INJECT_ENABLED="false",CLUSTER_ID="Kubernetes",CUSTOM_CA_CERT_NAME="higress-ca-root-cert",DEBUG_AUTH="false" [program:gateway] +directory=/ command=bash /usr/local/bin/start-gateway.sh priority=300 autostart=true @@ -48,9 +52,10 @@ startsecs=1 stdout_logfile=/var/log/higress/gateway.log stdout_logfile_maxbytes=10MB redirect_stderr=true -environment=JWT_POLICY="none",CA_ROOT_CA="/etc/certs/root-cert.pem",XDS_ROOT_CA="/etc/certs/root-cert.pem",PROV_CERT="/etc/certs",PILOT_XDS_SEND_TIMEOUT="60s",PROXY_XDS_VIA_AGENT="true",ENABLE_INGRESS_GATEWAY_SDS="false",POD_NAME="higress-gateway",POD_NAMESPACE="higress-system",ISTIO_META_CLUSTER_ID="Kubernetes",ISTIO_META_HTTP10="1",INSTANCE_NAME="higress-gateway" +environment=JWT_POLICY="none",CA_ROOT_CA="/etc/certs/root-cert.pem",XDS_ROOT_CA="/etc/certs/root-cert.pem",PROV_CERT="/etc/certs",PILOT_XDS_SEND_TIMEOUT="60s",PROXY_XDS_VIA_AGENT="true",ENABLE_INGRESS_GATEWAY_SDS="false",POD_NAME="higress-gateway",POD_NAMESPACE="higress-system",ISTIO_META_CLUSTER_ID="Kubernetes",ISTIO_META_HTTP10="1",INSTANCE_NAME="higress-gateway",LITE_METRICS="on" [program:console] +directory=/ command=bash /usr/local/bin/start-console.sh priority=900 autostart=true @@ -59,4 +64,52 @@ startsecs=1 stdout_logfile=/var/log/higress/console.log stdout_logfile_maxbytes=10MB redirect_stderr=true +environment= + +[program:prometheus] +directory=/ +command=bash /usr/local/bin/start-prometheus.sh +priority=2000 +autostart=true +autorestart=unexpected +startsecs=1 +stdout_logfile=/var/log/higress/prometheus.log +stdout_logfile_maxbytes=10MB +redirect_stderr=true +environment= + +[program:promtail] +directory=/ +command=bash /usr/local/bin/start-promtail.sh +priority=2100 +autostart=true +autorestart=unexpected +startsecs=1 +stdout_logfile=/var/log/higress/promtail.log +stdout_logfile_maxbytes=10MB +redirect_stderr=true +environment=HOSTNAME="higress-gateway" + +[program:loki] +directory=/ +command=bash /usr/local/bin/start-loki.sh +priority=2200 +autostart=true +autorestart=unexpected +startsecs=1 +stdout_logfile=/var/log/higress/loki.log +stdout_logfile_maxbytes=10MB +redirect_stderr=true +environment= + +[program:grafana] +directory=/ +command=bash /usr/local/bin/start-grafana.sh +priority=3000 +autostart=true +autorestart=unexpected +startsecs=1 +stdout_logfile=/var/log/higress/grafana.log +stdout_logfile_maxbytes=10MB +redirect_stderr=true environment= \ No newline at end of file diff --git a/bin/configure.sh b/bin/configure.sh index 324881d..2bc76d9 100755 --- a/bin/configure.sh +++ b/bin/configure.sh @@ -614,6 +614,10 @@ HIGRESS_CONTROLLER_TAG='${HIGRESS_CONTROLLER_TAG}' HIGRESS_PILOT_TAG='${HIGRESS_PILOT_TAG}' HIGRESS_GATEWAY_TAG='${HIGRESS_GATEWAY_TAG}' HIGRESS_CONSOLE_TAG='${HIGRESS_CONSOLE_TAG}' +PROMETHEUS_TAG='${PROMETHEUS_TAG}' +PROMTAIL_TAG='${PROMTAIL_TAG}' +LOKI_TAG='${LOKI_TAG}' +GRAFANA_TAG='${GRAFANA_TAG}' NACOS_HTTP_PORT='${NACOS_HTTP_PORT}' NACOS_GRPC_PORT='${NACOS_GRPC_PORT}' GATEWAY_HTTP_PORT='${GATEWAY_HTTP_PORT}' diff --git a/compose/.env b/compose/.env index fb2b40f..c202e50 100644 --- a/compose/.env +++ b/compose/.env @@ -8,11 +8,15 @@ NACOS_PASSWORD='' NACOS_DATA_ENC_KEY='' NACOS_SERVER_TAG='v2.2.3' HIGRESS_RUNNER_TAG='0.0.3' -HIGRESS_API_SERVER_TAG='0.0.16' +HIGRESS_API_SERVER_TAG='0.0.18' HIGRESS_CONTROLLER_TAG='2.0.4' HIGRESS_PILOT_TAG='2.0.4' HIGRESS_GATEWAY_TAG='2.0.4' HIGRESS_CONSOLE_TAG='1.4.6' +PROMETHEUS_TAG='v2.40.7' +PROMTAIL_TAG='2.9.4' +LOKI_TAG='2.9.4' +GRAFANA_TAG='9.3.6' NACOS_HTTP_PORT='8848' NACOS_GRPC_PORT='9848' GATEWAY_HTTP_PORT='80' diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index dd3a46c..787973e 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -203,6 +203,7 @@ services: - ./volumes/gateway/certs:/etc/certs:ro - ./volumes/gateway/podinfo:/etc/istio/pod:ro - ./volumes/gateway/istio/data:/var/lib/istio/data:rw + - ./volumes/gateway/log:/var/log/proxy:rw console: image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/console:${HIGRESS_CONSOLE_TAG} @@ -230,6 +231,126 @@ services: volumes: - ./volumes/kube:/home/higress/.kube:ro + prometheus: + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/prometheus:${PROMETHEUS_TAG} + command: + - "--config.file=/etc/prometheus/prometheus.yaml" + - "--web.external-url=/prometheus" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention=6h" + depends_on: + gateway: + condition: service_healthy + networks: + - higress-net + healthcheck: + test: + [ + "CMD", + "wget", + "--no-verbose", + "--tries=1", + "--spider", + "http://127.0.0.1:9090/prometheus/-/healthy/" + ] + interval: 1s + timeout: 3s + retries: 60 + restart: always + volumes: + - ./volumes/prometheus/config:/etc/prometheus:ro + - ./volumes/prometheus/data:/prometheus:rw + + promtail: + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/promtail:${PROMTAIL_TAG} + command: + - -config.file=/etc/promtail/promtail.yaml + env_file: + - ./env/promtail.env + depends_on: + gateway: + condition: service_healthy + loki: + condition: service_healthy + networks: + - higress-net + ports: + - "3101:3101/tcp" + healthcheck: + disable: true + # There is no curl or wget in the promtail image. We need to find another way to perform health check. + # test: + # [ + # "CMD", + # "curl", + # "-fk", + # "http://127.0.0.1:3101/ready" + # ] + # interval: 1s + # timeout: 3s + # retries: 60 + restart: always + volumes: + - ./volumes/promtail/config:/etc/promtail:ro + - ./volumes/promtail/data:/var/promtail:rw + - ./volumes/gateway/log:/var/log/proxy:ro + + loki: + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/loki:${LOKI_TAG} + command: + - -config.file=/etc/loki/config/config.yaml + - -target=all + networks: + - higress-net + ports: + - "3100:3100/tcp" + healthcheck: + test: + [ + "CMD", + "wget", + "--no-verbose", + "--tries=1", + "--spider", + "http://127.0.0.1:3100/ready" + ] + interval: 1s + timeout: 3s + retries: 60 + restart: always + volumes: + - ./volumes/loki/config:/etc/loki/config:ro + - ./volumes/loki/data:/var/loki:rw + + grafana: + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/grafana:${GRAFANA_TAG} + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_healthy + networks: + - higress-net + ports: + - "3000:3000/tcp" + healthcheck: + test: + [ + "CMD", + "wget", + "--no-verbose", + "--tries=1", + "--spider", + "http://127.0.0.1:3000/api/health" + ] + interval: 1s + timeout: 3s + retries: 60 + restart: always + volumes: + - ./volumes/grafana/config:/etc/grafana:ro + - ./volumes/grafana/lib:/var/lib/grafana:rw + postcheck: image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/runner:${HIGRESS_RUNNER_TAG} command: @@ -247,5 +368,14 @@ services: condition: service_healthy console: condition: service_healthy + # promtail doesn't support health check at the moment. + # promtail: + # condition: service_healthy + prometheus: + condition: service_healthy + loki: + condition: service_healthy + grafana: + condition: service_healthy volumes: - ./scripts:/workspace:ro diff --git a/compose/env/console.env b/compose/env/console.env index 9d6345d..a8b409f 100644 --- a/compose/env/console.env +++ b/compose/env/console.env @@ -2,4 +2,7 @@ JAVA_HOME=/usr/local/openjdk-18 LANG=C.UTF-8 HIGRESS_CONSOLE_KUBE_CONFIG=/home/higress/.kube/config HIGRESS_CONSOLE_CONTROLLER_SERVICE_HOST=pilot -HIGRESS_CONSOLE_CONTROLLER_ACCESS_TOKEN= \ No newline at end of file +HIGRESS_CONSOLE_CONTROLLER_ACCESS_TOKEN= +HIGRESS_CONSOLE_DASHBOARD_BASE_URL=http://grafana:3000/grafana +HIGRESS_CONSOLE_DASHBOARD_DATASOURCE_PROM_URL=http://prometheus:9090/prometheus +HIGRESS_CONSOLE_DASHBOARD_DATASOURCE_LOKI_URL=http://loki:3100 \ No newline at end of file diff --git a/compose/env/controller.env b/compose/env/controller.env index c1c7776..eade80f 100644 --- a/compose/env/controller.env +++ b/compose/env/controller.env @@ -1,2 +1,4 @@ CONTROLLER_KEEP_XDS_CONFIG_LABELS=false -CONTROLLER_KEEP_XDS_CONFIG_ANNOTATIONS=false \ No newline at end of file +CONTROLLER_KEEP_XDS_CONFIG_ANNOTATIONS=false +PILOT_ENABLE_GATEWAY_API=false +PILOT_ENABLE_ALPHA_GATEWAY_API=false \ No newline at end of file diff --git a/compose/env/gateway.env b/compose/env/gateway.env index e351277..87b3060 100644 --- a/compose/env/gateway.env +++ b/compose/env/gateway.env @@ -9,4 +9,5 @@ POD_NAME=higress-gateway POD_NAMESPACE=higress-system ISTIO_META_CLUSTER_ID=Kubernetes ISTIO_META_HTTP10=1 -INSTANCE_NAME=higress-gateway \ No newline at end of file +INSTANCE_NAME=higress-gateway +LITE_METRICS=on \ No newline at end of file diff --git a/compose/env/pilot.env b/compose/env/pilot.env index 6ab794a..0c99201 100644 --- a/compose/env/pilot.env +++ b/compose/env/pilot.env @@ -1,22 +1,36 @@ HIGRESS_CONTROLLER_SVC=controller -HIGRESS_CONTROLLER_PORT="15051" +HIGRESS_CONTROLLER_PORT=15051 REVISION=default JWT_POLICY=none POD_NAME=higress-pilot POD_NAMESPACE=higress-system PILOT_CERT_PROVIDER=istiod KUBECONFIG=/var/run/secrets/remote/config -PILOT_ENABLE_CROSS_CLUSTER_WORKLOAD_ENTRY="false" -PILOT_ENABLE_METADATA_EXCHANGE="false" -PILOT_SCOPE_GATEWAY_TO_NAMESPACE="true" -VALIDATION_ENABLED="false" -PILOT_TRACE_SAMPLING="1" -PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND="true" -PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND="true" -PILOT_ENABLE_ANALYSIS="false" -PILOT_ENABLE_GATEWAY_API=true +PILOT_ENABLE_CROSS_CLUSTER_WORKLOAD_ENTRY=false +PILOT_ENABLE_METADATA_EXCHANGE=false +PILOT_SCOPE_GATEWAY_TO_NAMESPACE=true +VALIDATION_ENABLED=false +PILOT_TRACE_SAMPLING=1 +PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND=true +PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND=true +PILOT_ENABLE_ANALYSIS=false +PILOT_ENABLE_GATEWAY_API=false +PILOT_ENABLE_ALPHA_GATEWAY_API=false PILOT_ENABLE_GATEWAY_API_STATUS=false PILOT_ENABLE_GATEWAY_API_DEPLOYMENT_CONTROLLER=false +PILOT_ENABLE_ALPN_FILTER=true +VALIDATION_WEBHOOK_CONFIG_NAME= +ISTIO_DUAL_STACK=false +ENABLE_OPTIMIZED_CONFIG_REBUILD=false +PILOT_ENABLE_K8S_SELECT_WORKLOAD_ENTRIES=false +DEFAULT_UPSTREAM_CONCURRENCY_THRESHOLD=10000 +ISTIO_GPRC_MAXRECVMSGSIZE=104857600 +ENBALE_SCOPED_RDS=true +ON_DEMAND_RDS=false +HOST_RDS_MERGE_SUBSET=false +PILOT_FILTER_GATEWAY_CLUSTER_CONFIG=true +PRIORITIZED_LEADER_ELECTION=false +INJECT_ENABLED=false CLUSTER_ID=Kubernetes CUSTOM_CA_CERT_NAME=higress-ca-root-cert DEBUG_AUTH=false diff --git a/compose/env/promtail.env b/compose/env/promtail.env new file mode 100644 index 0000000..494c17f --- /dev/null +++ b/compose/env/promtail.env @@ -0,0 +1 @@ +HOSTNAME=higress-gateway \ No newline at end of file diff --git a/compose/scripts/init.sh b/compose/scripts/init.sh index da6e675..2724d6d 100755 --- a/compose/scripts/init.sh +++ b/compose/scripts/init.sh @@ -299,6 +299,203 @@ higress="higress-system-higress-gateway" EOF mkdir -p $VOLUMES_ROOT/gateway/istio/data + + mkdir -p $VOLUMES_ROOT/gateway/log + touch $VOLUMES_ROOT/gateway/log/access.log +} + +initializePrometheus() { + echo "Initializing Prometheus configurations..." + + mkdir -p $VOLUMES_ROOT/prometheus/config && cd "$_" + cat <./prometheus.yaml +global: + scrape_interval: 15s + evaluation_interval: 15s +scrape_configs: + - job_name: 'prometheus' + metrics_path: /prometheus/metrics + static_configs: + - targets: ['localhost:9090'] + - job_name: 'gateway' + metrics_path: /stats/prometheus + static_configs: + - targets: ['gateway:15020'] + labels: + container: 'higress-gateway' + namespace: 'higress-system' + higress: 'higress-system-higress-gateway' + pod: 'higress' +EOF + + mkdir -p $VOLUMES_ROOT/prometheus/data + chmod a+rwx $VOLUMES_ROOT/prometheus/data +} + +initializePromtail() { + echo "Initializing Promtail configurations..." + + mkdir -p $VOLUMES_ROOT/promtail/config && cd "$_" + cat <./promtail.yaml +server: + log_level: info + http_listen_port: 3101 + +clients: +- url: http://loki:3100/loki/api/v1/push + +positions: + filename: /var/promtail/promtail-positions.yaml +target_config: + sync_period: 10s +scrape_configs: +- job_name: access-logs + static_configs: + - targets: + - localhost + labels: + __path__: /var/log/proxy/access.log + pipeline_stages: + - json: + expressions: + authority: + method: + path: + protocol: + request_id: + response_code: + response_flags: + route_name: + trace_id: + upstream_cluster: + upstream_host: + upstream_transport_failure_reason: + user_agent: + x_forwarded_for: + - labels: + authority: + method: + path: + protocol: + request_id: + response_code: + response_flags: + route_name: + trace_id: + upstream_cluster: + upstream_host: + upstream_transport_failure_reason: + user_agent: + x_forwarded_for: + - timestamp: + source: timestamp + format: RFC3339Nano +EOF + + mkdir -p $VOLUMES_ROOT/promtail/data + chmod a+rwx $VOLUMES_ROOT/promtail/data +} + +initializeLoki() { + echo "Initializing Loki configurations..." + + mkdir -p $VOLUMES_ROOT/loki/config && cd "$_" + cat <./config.yaml +auth_enabled: false +common: + compactor_address: 'loki' + path_prefix: /var/loki + replication_factor: 1 + storage: + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules +frontend: + scheduler_address: "" +frontend_worker: + scheduler_address: "" +index_gateway: + mode: ring +limits_config: + max_cache_freshness_per_query: 10m + reject_old_samples: true + reject_old_samples_max_age: 168h + split_queries_by_interval: 15m +memberlist: + join_members: + - loki +query_range: + align_queries_with_step: true +ruler: + storage: + type: local +runtime_config: + file: /etc/loki/config/runtime-config.yaml +schema_config: + configs: + - from: "2022-01-11" + index: + period: 24h + prefix: loki_index_ + object_store: filesystem + schema: v12 + store: boltdb-shipper +server: + http_listen_port: 3100 + grpc_listen_port: 9095 +storage_config: + hedging: + at: 250ms + max_per_second: 20 + up_to: 3 +tracing: + enabled: false +EOF + cat <./runtime-config.yaml +{} +EOF + + mkdir -p $VOLUMES_ROOT/loki/data/ + chmod a+rwx $VOLUMES_ROOT/loki/data/ +} + +initializeGrafana() { + echo "Initializing Grafana configurations..." + + mkdir -p $VOLUMES_ROOT/grafana/config && cd "$_" + cat <./grafana.ini +[server] +protocol=http +domain=localhost +root_url="%(protocol)s://%(domain)s/grafana" +serve_from_sub_path=true + +[auth] +disable_login_form=true +disable_signout_menu=true + +[auth.anonymous] +enabled=true +org_name=Main Org. +org_role=Viewer + +[users] +default_theme=light +viewers_can_edit=true + +[security] +allow_embedding=true +EOF + + mkdir -p $VOLUMES_ROOT/grafana/lib + chmod a+rwx $VOLUMES_ROOT/grafana/lib +} + +initializeO11y() { + initializePrometheus + initializePromtail + initializeLoki + initializeGrafana } initializeConfigStorage @@ -306,3 +503,4 @@ initializeApiServer initializeController initializePilot initializeGateway +initializeO11y diff --git a/compose/scripts/prepare.sh b/compose/scripts/prepare.sh index cdda390..f8cf8b7 100644 --- a/compose/scripts/prepare.sh +++ b/compose/scripts/prepare.sh @@ -33,7 +33,7 @@ checkConfigExists() { if [[ "$configGroupVersion" == *"/"* ]]; then uriPrefix="/apis" fi - local url; + local url if [ -z "$namespace" ]; then url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/${configType}/${configName}" else @@ -65,7 +65,7 @@ getConfig() { if [[ "$configGroupVersion" == *"/"* ]]; then uriPrefix="/apis" fi - local url; + local url if [ -z "$namespace" ]; then url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/${configType}/${configName}" else @@ -102,7 +102,7 @@ publishConfig() { if [[ "$configGroupVersion" == *"/"* ]]; then uriPrefix="/apis" fi - local url; + local url if [ -z "$namespace" ]; then url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/${configType}" else @@ -226,9 +226,22 @@ metadata: name: higress-config namespace: higress-system data: + higress: |- + downstream: + connectionBufferLimits: 32768 + http2: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65535 + maxConcurrentStreams: 100 + idleTimeout: 180 + maxRequestHeadersKb: 60 + routeTimeout: 0 + upstream: + connectionBufferLimits: 10485760 + idleTimeout: 10 mesh: |- accessLogEncoding: TEXT - accessLogFile: /dev/stdout + accessLogFile: /var/log/proxy/access.log accessLogFormat: | {"authority":"%REQ(X-ENVOY-ORIGINAL-HOST?:AUTHORITY)%","bytes_received":"%BYTES_RECEIVED%","bytes_sent":"%BYTES_SENT%","downstream_local_address":"%DOWNSTREAM_LOCAL_ADDRESS%","downstream_remote_address":"%DOWNSTREAM_REMOTE_ADDRESS%","duration":"%DURATION%","istio_policy_status":"%DYNAMIC_METADATA(istio.mixer:status)%","method":"%REQ(:METHOD)%","path":"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%","protocol":"%PROTOCOL%","request_id":"%REQ(X-REQUEST-ID)%","requested_server_name":"%REQUESTED_SERVER_NAME%","response_code":"%RESPONSE_CODE%","response_flags":"%RESPONSE_FLAGS%","route_name":"%ROUTE_NAME%","start_time":"%START_TIME%","trace_id":"%REQ(X-B3-TRACEID)%","upstream_cluster":"%UPSTREAM_CLUSTER%","upstream_host":"%UPSTREAM_HOST%","upstream_local_address":"%UPSTREAM_LOCAL_ADDRESS%","upstream_service_time":"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%","upstream_transport_failure_reason":"%UPSTREAM_TRANSPORT_FAILURE_REASON%","user_agent":"%REQ(USER-AGENT)%","x_forwarded_for":"%REQ(X-FORWARDED-FOR)%"} configSources: @@ -244,6 +257,9 @@ data: enableAutoMtls: false enablePrometheusMerge: true ingressControllerMode: "OFF" + mseIngressGlobalConfig: + enableH3: false + enableProxyProtocol: false protocolDetectionTimeout: 100ms rootNamespace: higress-system trustDomain: cluster.local @@ -396,8 +412,72 @@ EOF fi } +checkPrometheus() { + echo "Checking Prometheus configurations..." + + if [ ! -d "$VOLUMES_ROOT/prometheus/config/" ]; then + echo " The config folder of Prometheus is missing." + exit -1 + fi + if [ ! -f "$VOLUMES_ROOT/prometheus/config/prometheus.yaml" ] ; then + echo " Prometheus config file prometheus.yaml is missing." + exit -1 + fi +} + +checkPromtail() { + echo "Checking Promtail configurations..." + + if [ ! -d "$VOLUMES_ROOT/promtail/config/" ]; then + echo " The config folder of Promtail is missing." + exit -1 + fi + if [ ! -f "$VOLUMES_ROOT/promtail/config/promtail.yaml" ] ; then + echo " Promtail config file promtail.yaml is missing." + exit -1 + fi +} + +checkLoki() { + echo "Checking Loki configurations..." + + if [ ! -d "$VOLUMES_ROOT/loki/config/" ]; then + echo " The config folder of Loki is missing." + exit -1 + fi + if [ ! -f "$VOLUMES_ROOT/loki/config/config.yaml" ] ; then + echo " Loki config file config.yaml is missing." + exit -1 + fi + if [ ! -f "$VOLUMES_ROOT/loki/config/runtime-config.yaml" ] ; then + echo " Loki config file runtime-config.yaml is missing." + exit -1 + fi +} + +checkGrafana() { + echo "Checking Grafana configurations..." + + if [ ! -d "$VOLUMES_ROOT/grafana/config/" ]; then + echo " The config folder of Grafana is missing." + exit -1 + fi + if [ ! -f "$VOLUMES_ROOT/grafana/config/grafana.ini" ] ; then + echo " Grafana config file grafana.ini is missing." + exit -1 + fi +} + +checkO11y() { + checkPrometheus + checkPromtail + checkLoki + checkGrafana +} + checkStorage checkPilot checkGateway checkConsole checkGatewayApi +checkO11y diff --git a/src/apiserver/.gitignore b/src/apiserver/.gitignore index a7c36e3..643659e 100644 --- a/src/apiserver/.gitignore +++ b/src/apiserver/.gitignore @@ -22,6 +22,7 @@ vendor/ # editor and IDE paraphernalia .idea +.run *.swp *.swo *~ diff --git a/src/apiserver/.run/API Server.run.xml b/src/apiserver/.run/API Server.run.xml deleted file mode 100644 index 7a7fdf7..0000000 --- a/src/apiserver/.run/API Server.run.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file