Bugfix/cldsrv 473 fix veeam get capacity

.github/actions/setup-ci/action.yaml

@@ -38,3 +38,9 @@ runs:
       run: |
         sudo apt-get install -y libdigest-hmac-perl
         pip install 's3cmd==2.3.0'
+    - name: fix sproxyd.conf permissions
+      shell: bash
+      run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf
+    - name: ensure fuse kernel module is loaded (for sproxyd)
+      shell: bash
+      run: sudo modprobe fuse

.github/docker/docker-compose.yaml

@@ -78,3 +78,10 @@ services:
     network_mode: "host"
     profiles: ['ceph']
     image: ghcr.io/scality/cloudserver/ci-ceph 
+  sproxyd:
+    network_mode: "host"
+    profiles: ['sproxyd']
+    image: sproxyd-standalone
+    build: ./sproxyd
+    user: 0:0
+    privileged: yes

.github/docker/mongodb/Dockerfile

@@ -1,4 +1,4 @@
-FROM mongo:4.2.24
+FROM mongo:4.4.21
 
 ENV USER=scality \
     HOME_DIR=/home/scality \

.github/docker/sproxyd/Dockerfile

@@ -0,0 +1,3 @@
+FROM registry.scality.com/federation/sproxyd:7.10.6.8
+ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
+RUN chown root:root /conf/sproxyd0.conf

.github/docker/sproxyd/conf/fastcgi_params

@@ -0,0 +1,26 @@
+fastcgi_param  QUERY_STRING       $query_string;
+fastcgi_param  REQUEST_METHOD     $request_method;
+fastcgi_param  CONTENT_TYPE       $content_type;
+fastcgi_param  CONTENT_LENGTH     $content_length;
+
+#fastcgi_param  SCRIPT_NAME        $fastcgi_script_name;
+fastcgi_param  SCRIPT_NAME        /var/www;
+fastcgi_param  PATH_INFO          $document_uri;
+
+fastcgi_param  REQUEST_URI        $request_uri;
+fastcgi_param  DOCUMENT_URI       $document_uri;
+fastcgi_param  DOCUMENT_ROOT      $document_root;
+fastcgi_param  SERVER_PROTOCOL    $server_protocol;
+fastcgi_param  HTTPS              $https if_not_empty;
+
+fastcgi_param  GATEWAY_INTERFACE  CGI/1.1;
+fastcgi_param  SERVER_SOFTWARE    nginx/$nginx_version;
+
+fastcgi_param  REMOTE_ADDR        $remote_addr;
+fastcgi_param  REMOTE_PORT        $remote_port;
+fastcgi_param  SERVER_ADDR        $server_addr;
+fastcgi_param  SERVER_PORT        $server_port;
+fastcgi_param  SERVER_NAME        $server_name;
+
+# PHP only, required if PHP was built with --enable-force-cgi-redirect
+fastcgi_param  REDIRECT_STATUS    200;

.github/docker/sproxyd/conf/nginx.conf

@@ -0,0 +1,88 @@
+worker_processes 1;
+error_log  /logs/error.log;
+user root root;
+events {
+    worker_connections  1000;
+    reuse_port on;
+    multi_accept on;
+}
+worker_rlimit_nofile    20000;
+http {
+    root /var/www/;
+    upstream sproxyds {
+        least_conn;
+        keepalive 40;
+        server 127.0.0.1:20000;
+    }
+    server {
+        client_max_body_size 0;
+        client_body_timeout 150;
+        client_header_timeout 150;
+        postpone_output 0;
+        client_body_postpone_size 0;
+        keepalive_requests 1100;
+        keepalive_timeout 300s;
+        server_tokens off;
+        default_type application/octet-stream;
+        gzip off;
+        tcp_nodelay on;
+        tcp_nopush  on;
+        sendfile on;
+        listen 81;
+        server_name localhost;
+        rewrite ^/arc/(.*)$ /dc1/$1 permanent;
+        location ~* ^/proxy/(.*)$ {
+            rewrite ^/proxy/(.*)$ /$1 last;
+        }
+        allow 127.0.0.1;
+
+        deny all;
+        set $usermd '-';
+        set $sentusermd '-';
+    	set $elapsed_ms '-';
+    	set $now '-';
+    	log_by_lua '
+    		if not(ngx.var.http_x_scal_usermd == nil) and  string.len(ngx.var.http_x_scal_usermd) > 2 then
+    			ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3)
+    		end
+                    if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then
+                            ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3)
+                    end
+    		local elapsed_ms = tonumber(ngx.var.request_time)
+    		if not (  elapsed_ms == nil) then
+    			elapsed_ms = elapsed_ms * 1000
+    			ngx.var.elapsed_ms = tostring(elapsed_ms)
+    		end
+    		local time = tonumber(ngx.var.msec) * 1000
+    		ngx.var.now = time
+    	';
+        log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",'
+                       '"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,'
+                       '"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,'
+		       '"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",'
+		       '"contentType":"$content_type","s3Address":"$remote_addr",'
+		       '"requestUserMd":"$usermd","responseUserMd":"$sentusermd",'
+                       '"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",'
+		       '"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",'
+		       '"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",'
+                       '"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,'
+                       '"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }';
+        access_log /dev/stdout irm;
+    	error_log  /dev/stdout error;
+        location / {
+            proxy_request_buffering off;
+            fastcgi_request_buffering off;
+            fastcgi_no_cache 1;
+            fastcgi_cache_bypass 1;
+            fastcgi_buffering off;
+            fastcgi_ignore_client_abort on;
+            fastcgi_keep_conn on;
+            include fastcgi_params;
+            fastcgi_pass sproxyds;
+            fastcgi_next_upstream error timeout;
+            fastcgi_send_timeout 285s;
+            fastcgi_read_timeout 285s;
+        }
+    }
+}
+

.github/docker/sproxyd/conf/sproxyd0.conf

@@ -0,0 +1,12 @@
+{
+    "general": {
+        "ring": "DATA",
+        "port": 20000,
+        "syslog_facility": "local0"
+    },
+    "ring_driver:0": {
+        "alias": "dc1",
+        "type": "local",
+        "queue_path": "/tmp/ring-objs"
+    },
+}

.github/docker/sproxyd/conf/supervisord.conf

@@ -0,0 +1,43 @@
+[supervisord]
+nodaemon = true
+loglevel = info
+logfile  = %(ENV_LOG_DIR)s/supervisord.log
+pidfile  = %(ENV_SUP_RUN_DIR)s/supervisord.pid
+logfile_maxbytes = 20MB
+logfile_backups = 2
+
+[unix_http_server]
+file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[program:nginx]
+directory=%(ENV_SUP_RUN_DIR)s
+command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'"
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart=true
+autostart=true
+user=root
+
+[program:sproxyd]
+directory=%(ENV_SUP_RUN_DIR)s
+process_name=%(program_name)s-%(process_num)s
+numprocs=1
+numprocs_start=0
+command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+redirect_stderr=true
+autorestart=true
+autostart=true
+user=root

.github/workflows/tests.yaml

@@ -179,15 +179,22 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v3
+    - name: Login to Registry
+      uses: docker/login-action@v2
+      with:
+        registry: registry.scality.com
+        username: ${{ secrets.REGISTRY_LOGIN }}
+        password: ${{ secrets.REGISTRY_PASSWORD }}
     - name: Setup CI environment
       uses: ./.github/actions/setup-ci
     - name: Setup CI services
-      run: docker-compose up -d
+      run: docker-compose --profile sproxyd up -d
       working-directory: .github/docker
     - name: Run multiple backend test
       run: |-
         set -o pipefail;
         bash wait_for_local_port.bash 8000 40
+        bash wait_for_local_port.bash 81 40
         yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
       env:
         S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json

lib/routes/veeam/get.js

@@ -18,7 +18,7 @@ function getVeeamFile(request, response, bucketMd, log) {
         return responseXMLBody(errors.NoSuchBucket, null, response, log);
     }
     if ('tagging' in request.query) {
-        return respondWithData(request, response, log, request.bucketName,
+        return respondWithData(request, response, log, bucketMd,
             buildHeadXML('<Tagging><TagSet></TagSet></Tagging>'));
     }
     return metadata.getBucket(request.bucketName, log, (err, data) => {
@@ -38,7 +38,7 @@ function getVeeamFile(request, response, bucketMd, log) {
         const builder = new xml2js.Builder({
             headless: true,
         });
-        return respondWithData(request, response, log, request.bucketName,
+        return respondWithData(request, response, log, data,
             buildHeadXML(builder.buildObject(fileToBuild.value)), modified);
     });
 }

lib/routes/veeam/head.js

@@ -35,7 +35,7 @@ function headVeeamFile(request, response, bucketMd, log) {
             headless: true,
         });
         const dataBuffer = Buffer.from(buildHeadXML(builder.buildObject(fileToBuild)));
-        return responseContentHeaders(null, {}, getResponseHeader(request, request.bucketName,
+        return responseContentHeaders(null, {}, getResponseHeader(request, data,
             dataBuffer, modified, log), response, log);
     });
 }

lib/routes/veeam/list.js

@@ -118,13 +118,13 @@ function listVeeamFiles(request, response, bucketMd, log) {
             });
             const dataBuffer = Buffer.from(buildHeadXML(builder.buildObject(file)));
             filesToBuild.push({
-                ...getResponseHeader(request, request.bucketName,
+                ...getResponseHeader(request, data,
                     dataBuffer, lastModified, log),
                 name: file.name,
             });
         });
         // When `versions` is present, listing should return a versioned list
-        return respondWithData(request, response, log, request.bucketName,
+        return respondWithData(request, response, log, data,
             buildXMLResponse(request, filesToBuild, 'versions' in request.query));
     });
 }

lib/routes/veeam/utils.js

@@ -72,7 +72,7 @@ function buildHeadXML(xmlContent) {
 /**
  * Get response headers for the object
  * @param {object} request - incoming request
- * @param {string} bucket - bucket name
+ * @param {BucketInfo} bucket - bucket
  * @param {string} dataBuffer - data to send as a buffer
  * @param {date} [lastModified] - last modified date of the value
  * @param {object} log - logging object
@@ -101,7 +101,7 @@ function getResponseHeader(request, bucket, dataBuffer, lastModified, log) {
  * @param {object} request - incoming request
  * @param {object} response - response object
  * @param {object} log - logging object
- * @param {string} bucket - bucket name
+ * @param {BucketInfo} bucket - bucket info
  * @param {string} data - data to send
  * @param {date} [lastModified] - last modified date of the value
  * @returns {undefined} -

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenko/cloudserver",
-  "version": "8.7.32",
+  "version": "8.8.6",
   "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol",
   "main": "index.js",
   "engines": {

tests/locationConfig/locationConfigTests.json

@@ -35,6 +35,19 @@
         "legacyAwsBehavior": false,
         "details": {}
     },
+    "scality-internal-sproxyd": {
+        "type": "scality",
+        "objectId": "scality-internal-sproxyd",
+        "legacyAwsBehavior": false,
+        "details": {
+            "connector": {
+                "sproxyd": {
+                    "bootstrap": ["127.0.0.1:81"],
+                    "path": "/proxy/dc1/"
+                }
+            }
+        }
+    },
     "awsbackendencryption": {
         "type": "aws_s3",
         "objectId": "awsbackendencryption",

tests/multipleBackend/backendHealthcheckResponse.js

@@ -14,13 +14,17 @@ const log = new DummyRequestLogger();
 const locConstraints = Object.keys(config.locationConstraints);
 const azureClient = getAzureClient();
 
-describe.skip('Healthcheck response', () => {
+describe('Healthcheck response', () => {
     it('should return result for every location constraint in ' +
     'locationConfig and every external locations with flightCheckOnStartUp ' +
     'set to true', done => {
         clientCheck(true, log, (err, results) => {
             const resultKeys = Object.keys(results);
             locConstraints.forEach(constraint => {
+                if (constraint === 'location-dmf-v1') {
+                    // FIXME: location-dmf-v1 is not in results, see CLDSRV-440
+                    return;
+                }
                 assert(resultKeys.includes(constraint), `constraint: ${constraint} not in results: ${resultKeys}`);
             });
             done();
@@ -40,6 +44,10 @@ describe.skip('Healthcheck response', () => {
         clientCheck(false, log, (err, results) => {
             assert.notStrictEqual(results.length, locConstraints.length);
             locConstraints.forEach(constraint => {
+                if (constraint === 'location-dmf-v1') {
+                    // FIXME: location-dmf-v1 is not in results, see CLDSRV-440
+                    return;
+                }
                 if (Object.keys(results).indexOf(constraint) === -1) {
                     const locationType = config
                         .locationConstraints[constraint].type;
@@ -52,7 +60,8 @@ describe.skip('Healthcheck response', () => {
         });
     });
 
-    describe('Azure container creation', () => {
+    // FIXME: does not pass, see CLDSRV-441
+    describe.skip('Azure container creation', () => {
         const containerName =
             getAzureContainerName(azureLocationNonExistContainer);
 

tests/multipleBackend/multipartUpload.js

@@ -325,7 +325,7 @@ function abortMultipleMpus(backendsInfo, callback) {
     });
 }
 
-describe.skip('Multipart Upload API with AWS Backend', function mpuTestSuite() {
+describe('Multipart Upload API with AWS Backend', function mpuTestSuite() {
     this.timeout(60000);
 
     beforeEach(done => {

tests/multipleBackend/objectCopy.js

@@ -72,7 +72,7 @@ function copySetup(params, cb) {
     ], err => cb(err));
 }
 
-describe.skip('ObjectCopy API with multiple backends', () => {
+describe('ObjectCopy API with multiple backends', () => {
     before(() => {
         cleanup();
     });