Skip to content

Commit

Permalink
Merge pull request #286 from inbo/export-direct
Browse files Browse the repository at this point in the history 
export direct
  • Loading branch information
@mainlyIt
mainlyIt authored Jan 4, 2025
2 parents 599757b + de33f56 commit 1bfb11b
Show file tree
Hide file tree
Showing 5 changed files with 422 additions and 226 deletions.
222 changes: 184 additions & 38 deletions nginx.conf
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
worker_processes auto; # Changed from 1 to auto for better performance
# Use all available CPU cores
worker_processes auto;

# Configure worker settings
worker_rlimit_nofile 65535;
timer_resolution 100ms;
pcre_jit on;

events {
worker_connections 4096;
Expand All @@ -7,110 +13,250 @@ events {
}

http {
include mime.types;
default_type application/octet-stream;
# Basic settings
include mime.types;
default_type application/octet-stream;
server_tokens off;

# Character encoding
charset utf-8;
charset_types text/css text/plain text/vnd.wap.wml text/javascript text/markdown text/calendar text/x-component text/x-cross-domain-policy;

# Optimization for file handling
sendfile on;
tcp_nopush on;
tcp_nodelay on;
aio threads;
directio 512;

# Buffer size optimization
client_body_buffer_size 128k;
client_max_body_size 0;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;

# Timeouts
client_body_timeout 300s;
client_header_timeout 300s;
send_timeout 600s;
keepalive_timeout 650s;
keepalive_requests 1000;

# HTTP/2 specific settings
http2_max_field_size 16k;
http2_max_header_size 32k;
http2_max_requests 1000;
http2_idle_timeout 5m;
http2_idle_timeout 300s;

# Global timeout settings
proxy_connect_timeout 600;
proxy_send_timeout 600;
proxy_read_timeout 1800;
send_timeout 600;
keepalive_timeout 650;

sendfile on;
tcp_nopush on;
tcp_nodelay on;

# Buffering settings for large responses
# Proxy settings
proxy_connect_timeout 600s;
proxy_send_timeout 600s;
proxy_read_timeout 1800s;
proxy_buffer_size 128k;
proxy_buffers 8 256k;
proxy_busy_buffers_size 256k;
proxy_temp_file_write_size 256k;
proxy_max_temp_file_size 0;
proxy_request_buffering on;
proxy_http_version 1.1;
proxy_buffering on;

# Compression settings
gzip on;
gzip_comp_level 5;
gzip_min_length 256;
gzip_proxied any;
gzip_vary on;
gzip_types
application/javascript
application/json
application/xml
text/css
text/plain
text/xml
text/javascript;

# File cache settings
open_file_cache max=1000 inactive=20s;
open_file_cache_valid 30s;
open_file_cache_min_uses 2;
open_file_cache_errors on;

# Common security headers
add_header X-Content-Type-Options nosniff always;
add_header X-Frame-Options SAMEORIGIN always;
add_header X-XSS-Protection "1; mode=block" always;
add_header Referrer-Policy strict-origin-when-cross-origin always;

# UAT Server Configuration
server {
listen 80 http2;
server_name uat.vespadb.be;

# Increase client body size limit
client_max_body_size 0; # Disabled limit for large files

# Static files
location /static/ {
alias /workspaces/vespadb/collected_static/;
expires 30d;
access_log off;
add_header Cache-Control "public, no-transform";
}

location /media/ {
alias /workspaces/vespadb/media/;
expires 30d;
access_log off;
add_header Cache-Control "public, no-transform";
}

# API and main application
location / {
proxy_pass http://127.0.0.1:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";

# CORS headers for API
add_header 'Access-Control-Allow-Origin' $http_origin always;
add_header 'Access-Control-Allow-Credentials' 'true' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since' always;

# Handle OPTIONS method for CORS
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' $http_origin always;
add_header 'Access-Control-Allow-Credentials' 'true' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since' always;
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Type' 'text/plain charset=UTF-8';
add_header 'Content-Length' 0;
return 204;
}
}

# HTTP/2 specific
proxy_http_version 1.1;
# Special handling for export endpoint
location /api/observations/export_direct {
proxy_pass http://127.0.0.1:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";

# Timeouts
proxy_connect_timeout 600s;
proxy_send_timeout 600s;
# Extended timeouts for long-running exports
proxy_read_timeout 1800s;
proxy_connect_timeout 600s;
proxy_send_timeout 600s;

# Buffer settings for large files
proxy_buffering on;
proxy_request_buffering on;
# Buffer settings for large responses
proxy_buffer_size 128k;
proxy_buffers 8 256k;
proxy_busy_buffers_size 256k;

# CORS headers
add_header 'Access-Control-Allow-Origin' $http_origin always;
add_header 'Access-Control-Allow-Credentials' 'true' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since' always;

if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' $http_origin always;
add_header 'Access-Control-Allow-Credentials' 'true' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since' always;
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Type' 'text/plain charset=UTF-8';
add_header 'Content-Length' 0;
return 204;
}
}
}

# Production Server Configuration
server {
listen 80 http2;
server_name data.vespawatch.be;

# Increase client body size limit
client_max_body_size 0; # Disabled limit for large files

# Static files
location /static/ {
alias /workspaces/vespadb/collected_static/;
expires 30d;
access_log off;
add_header Cache-Control "public, no-transform";
}

location /media/ {
alias /workspaces/vespadb/media/;
expires 30d;
access_log off;
add_header Cache-Control "public, no-transform";
}

# API and main application
location / {
proxy_pass http://127.0.0.1:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";

# CORS headers for API
add_header 'Access-Control-Allow-Origin' $http_origin always;
add_header 'Access-Control-Allow-Credentials' 'true' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since' always;

# Handle OPTIONS method for CORS
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' $http_origin always;
add_header 'Access-Control-Allow-Credentials' 'true' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since' always;
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Type' 'text/plain charset=UTF-8';
add_header 'Content-Length' 0;
return 204;
}
}

# HTTP/2 specific
proxy_http_version 1.1;
# Special handling for export endpoint
location /api/observations/export_direct {
proxy_pass http://127.0.0.1:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";

# Timeouts
# Extended timeouts for long-running exports
proxy_read_timeout 1800s;
proxy_connect_timeout 600s;
proxy_send_timeout 600s;
proxy_read_timeout 1800s;

# Buffer settings for large files
proxy_buffering on;
proxy_request_buffering on;
# Buffer settings for large responses
proxy_buffer_size 128k;
proxy_buffers 8 256k;
proxy_busy_buffers_size 256k;

# CORS headers
add_header 'Access-Control-Allow-Origin' $http_origin always;
add_header 'Access-Control-Allow-Credentials' 'true' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since' always;

if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' $http_origin always;
add_header 'Access-Control-Allow-Credentials' 'true' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since' always;
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Type' 'text/plain charset=UTF-8';
add_header 'Content-Length' 0;
return 204;
}
}
}
}
}
Loading

0 comments on commit 1bfb11b

Please sign in to comment.