From 61bf1f114929e5cbd0a373d1e521de97e1fba6cf Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 12 Sep 2022 13:58:29 +0200 Subject: [PATCH 001/325] Revert "Bridge: return message and not status on failed getlock" This reverts commit d91171c1d96f7c3ed6def47c149395876c7bae9c. --- src/bridge/wopiclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge/wopiclient.py b/src/bridge/wopiclient.py index 2f773514..c0e09365 100644 --- a/src/bridge/wopiclient.py +++ b/src/bridge/wopiclient.py @@ -85,7 +85,7 @@ def getlock(wopisrc, acctok): res = request(wopisrc, acctok, 'POST', headers={'X-Wopi-Override': 'GET_LOCK'}) if res.status_code != http.client.OK: # lock got lost or any other error - raise InvalidLock(res.content.decode()) + raise InvalidLock(res.status_code) # the lock is expected to be a JSON dict, see generatelock() return json.loads(res.headers['X-WOPI-Lock']) except (ValueError, KeyError, json.decoder.JSONDecodeError) as e: From 7a037dfe5c3b890a7e0179c7a662367280f0f48c Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 12 Sep 2022 15:05:29 +0200 Subject: [PATCH 002/325] Fixed new header for bridged apps --- src/bridge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 2f5c48f5..c01de49c 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -26,7 +26,7 @@ BRIDGE_EXT_PLUGINS = {'md': 'codimd', 'txt': 'codimd', 'zmd': 'codimd', 'epd': 'etherpad', 'zep': 'etherpad'} # The header that bridged apps are expected to send to the save endpoint -BRIDGED_APP_HEADER = 'X-EFSS-Bridged-App' +BRIDGED_APP_HEADER = 'X-Efss-Bridged-App' # a standard message to be displayed by the app when some content might be lost: this would only # appear in case of uncaught exceptions or bugs handling the webhook callbacks From 6b01c04126c4b71707bbf0b32471c6a011148e5b Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 13 Sep 2022 10:38:53 +0200 Subject: [PATCH 003/325] Bridged apps: relaxed check for app names --- src/bridge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index c01de49c..143f08e6 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -128,7 +128,7 @@ def _getappnamebyaddr(remoteaddr): def _validateappname(appname): '''Return the plugin's appname if one of the registered plugins matches (case-insensitive) the given appname''' for p in WB.plugins.values(): - if appname.lower() == p.appname.lower(): + if appname.lower() in p.appname.lower(): return p.appname raise ValueError From c71f693810db90f230355a6900e8c0027a3e24f0 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 13 Sep 2022 16:17:26 +0200 Subject: [PATCH 004/325] Temporary workaround for CodiMD --- src/bridge/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 143f08e6..51fd6312 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -246,9 +246,11 @@ def appsave(docid): (flask.request.remote_addr, flask.request.headers, flask.request.args, e)) return wopic.jsonify('Missing metadata, could not save. %s' % RECOVER_MSG), http.client.BAD_REQUEST except ValueError as e: - WB.log.error('msg="BridgeSave: unknown application" address="%s" headers="%s" args="%s"' % - (flask.request.remote_addr, flask.request.headers, flask.request.args)) - return wopic.jsonify('Unknown application, could not save. %s' % RECOVER_MSG), http.client.UNAUTHORIZED + WB.log.error('msg="BridgeSave: unknown application" address="%s" appHeader="%s" args="%s"' % + (flask.request.remote_addr, flask.request.headers.get(BRIDGED_APP_HEADER), flask.request.args)) + # temporary override + appname = 'CodiMD' + #return wopic.jsonify('Unknown application, could not save. %s' % RECOVER_MSG), http.client.UNAUTHORIZED # decide whether to notify the save thread donotify = isclose or wopisrc not in WB.openfiles or WB.openfiles[wopisrc]['lastsave'] < time.time() - WB.saveinterval From 8081a79e94187ecaf71c10f5bdd9496eb4bfafc2 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 14 Sep 2022 19:12:39 +0200 Subject: [PATCH 005/325] Further logging to chase unresolved conflicts --- src/core/wopi.py | 31 +++++++++++++++++-------------- src/core/wopiutils.py | 17 ++++++++++++----- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index ba34f1fb..f26aa9e3 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -173,7 +173,7 @@ def setLock(fileid, reqheaders, acctok): else: savetime = None if not savetime: - return utils.makeConflictResponse(op, acctok['userid'], None, lock, oldLock, fn, + return utils.makeConflictResponse(op, acctok['userid'], None, lock, oldLock, acctok['endpoint'], fn, 'The file was not locked' + ' and got modified' if validateTarget else '') # now create an "external" lock if required @@ -207,7 +207,8 @@ def setLock(fileid, reqheaders, acctok): log.warning('msg="Valid LibreOffice lock found, denying WOPI lock" lockop="%s" filename="%s" holder="%s"' % (op.title(), fn, lockholder if lockholder else retrievedlolock)) reason = 'File locked by ' + ((lockholder + ' via LibreOffice') if lockholder else 'a LibreOffice user') - return utils.makeConflictResponse(op, acctok['userid'], 'External App', lock, oldLock, fn, reason) + return utils.makeConflictResponse(op, acctok['userid'], 'External App', lock, oldLock, + acctok['endpoint'], fn, reason) # else it's our previous lock or it had expired: all right, move on else: # any other error is logged but not raised as this is optimistically not blocking WOPI operations @@ -220,8 +221,8 @@ def setLock(fileid, reqheaders, acctok): # LOCK or REFRESH_LOCK: atomically set the lock to the given one, including the expiration time, # and return conflict response if the file was already locked st.setlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(lock)) - log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" lock="%s"' % - (op.title(), fn, flask.request.args['access_token'][-20:], lock)) + log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s"' % + (op.title(), fn, flask.request.args['access_token'][-20:], flask.request.headers.get('X-WOPI-SessionId'), lock)) # on first lock, set an xattr with the current time for later conflicts checking try: @@ -253,15 +254,15 @@ def setLock(fileid, reqheaders, acctok): if retrievedLock and not utils.compareWopiLocks(retrievedLock, (oldLock if oldLock else lock)): # lock mismatch, the WOPI client is supposed to acknowledge the existing lock # or deny write access to the file - return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, fn, + return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, acctok['endpoint'], fn, 'The file is locked by %s' % (lockHolder if lockHolder != 'wopi' else 'another online editor')) # else it's our own lock, refresh it and return try: st.refreshlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(lock)) - log.info('msg="Successfully refreshed" lockop="%s" filename="%s" token="%s" lock="%s"' % - (op.title(), fn, flask.request.args['access_token'][-20:], lock)) + log.info('msg="Successfully refreshed" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s"' % + (op.title(), fn, flask.request.args['access_token'][-20:], flask.request.headers.get('X-WOPI-SessionId'), lock)) # else we don't need to refresh it again resp = flask.Response() resp.status_code = http.client.OK @@ -309,7 +310,7 @@ def unlock(fileid, reqheaders, acctok): retrievedLock, _ = utils.retrieveWopiLock(fileid, 'UNLOCK', lock, acctok) if not utils.compareWopiLocks(retrievedLock, lock): return utils.makeConflictResponse('UNLOCK', acctok['userid'], retrievedLock, lock, 'NA', - acctok['filename'], 'Lock mismatch') + acctok['endpoint'], acctok['filename'], 'Lock mismatch') # OK, the lock matches, remove it try: # validate that the underlying file is still there @@ -387,7 +388,8 @@ def putRelative(fileid, reqheaders, acctok): retrievedTargetLock, _ = utils.retrieveWopiLock(fileid, 'PUT_RELATIVE', None, acctok, overridefn=relTarget) # deny if lock is valid or if overwriteTarget is False if not overwriteTarget or retrievedTargetLock: - return utils.makeConflictResponse('PUT_RELATIVE', acctok['userid'], retrievedTargetLock, 'NA', 'NA', relTarget, { + return utils.makeConflictResponse('PUT_RELATIVE', acctok['userid'], retrievedTargetLock, 'NA', 'NA', + acctok['endpoint'], relTarget, { 'message': 'Target file already exists', # specs (the WOPI validator) require these to be populated with valid values 'Name': os.path.basename(relTarget), @@ -430,8 +432,8 @@ def deleteFile(fileid, _reqheaders_unused, acctok): retrievedLock, _ = utils.retrieveWopiLock(fileid, 'DELETE', '', acctok) if retrievedLock is not None: # file is locked and cannot be deleted - return utils.makeConflictResponse('DELETE', acctok['userid'], retrievedLock, 'NA', 'NA', acctok['filename'], - 'Cannot delete a locked file') + return utils.makeConflictResponse('DELETE', acctok['userid'], retrievedLock, 'NA', 'NA', + acctok['endpoint'], acctok['filename'], 'Cannot delete a locked file') try: st.removefile(acctok['endpoint'], acctok['filename'], acctok['userid']) return 'OK', http.client.OK @@ -455,7 +457,8 @@ def renameFile(fileid, reqheaders, acctok): lock = reqheaders.get('X-WOPI-Lock') retrievedLock, _ = utils.retrieveWopiLock(fileid, 'RENAMEFILE', lock, acctok) if retrievedLock is not None and not utils.compareWopiLocks(retrievedLock, lock): - return utils.makeConflictResponse('RENAMEFILE', acctok['userid'], retrievedLock, lock, 'NA', acctok['filename']) + return utils.makeConflictResponse('RENAMEFILE', acctok['userid'], retrievedLock, lock, 'NA', + acctok['endpoint'], acctok['filename']) try: # the destination name comes without base path and typically without extension targetName = os.path.dirname(acctok['filename']) + os.path.sep + targetName \ @@ -516,8 +519,8 @@ def putFile(fileid, acctok): lock = flask.request.headers['X-WOPI-Lock'] retrievedLock, lockHolder = utils.retrieveWopiLock(fileid, 'PUTFILE', lock, acctok) if retrievedLock is None: - return utils.makeConflictResponse('PUTFILE', acctok['userid'], retrievedLock, lock, 'NA', acctok['filename'], - 'Cannot overwrite unlocked file') + return utils.makeConflictResponse('PUTFILE', acctok['userid'], retrievedLock, lock, 'NA', + acctok['endpoint'], acctok['filename'], 'Cannot overwrite unlocked file') if not utils.compareWopiLocks(retrievedLock, lock): log.warning('msg="Forcing conflict based on external lock" user="%s" filename="%s" token="%s"' % (acctok['userid'][-20:], acctok['filename'], flask.request.args['access_token'][-20:])) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index dc6a33ab..99633ad1 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -330,7 +330,7 @@ def compareWopiLocks(lock1, lock2): return False -def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, filename, reason=None): +def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, endpoint, filename, reason=None): '''Generates and logs an HTTP 409 response in case of locks conflict''' resp = flask.Response(mimetype='application/json') resp.headers['X-WOPI-Lock'] = retrievedlock if retrievedlock else '' @@ -341,10 +341,15 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, filename reason = {'message': reason} resp.headers['X-WOPI-LockFailureReason'] = reason['message'] resp.data = json.dumps(reason) + savetime = st.getxattr(endpoint, filename, user, LASTSAVETIMEKEY) + if savetime: + savetime = int(savetime) + else: + savetime = 0 log.warning('msg="Returning conflict" lockop="%s" user="%s" filename="%s" token="%s" sessionId="%s" lock="%s" ' - 'oldlock="%s" retrievedlock="%s" reason="%s"' % + 'oldlock="%s" retrievedlock="%s" fileage="%s" reason="%s"' % (operation.title(), user, filename, flask.request.args['access_token'][-20:], - flask.request.headers.get('X-WOPI-SessionId'), lock, oldlock, retrievedlock, + flask.request.headers.get('X-WOPI-SessionId'), lock, oldlock, retrievedlock, time.time() - savetime, (reason['message'] if reason else 'NA'))) return resp @@ -386,12 +391,14 @@ def storeAfterConflict(acctok, retrievedlock, lock, reason): storeForRecovery(flask.request.get_data(), acctok['username'], newname, flask.request.args['access_token'][-20:], dorecovery) # conflict file was stored on recovery space, tell user (but reason is advisory...) - return makeConflictResponse('PUTFILE', acctok['userid'], retrievedlock, lock, 'NA', acctok['filename'], + return makeConflictResponse('PUTFILE', acctok['userid'], retrievedlock, lock, 'NA', + acctok['endpoint'], acctok['filename'], reason + ', please contact support to recover it') # otherwise, conflict file was saved to user space but we still use a CONFLICT response # as it is better handled by the app to signal the issue to the user - return makeConflictResponse('PUTFILE', acctok['userid'], retrievedlock, lock, 'NA', acctok['filename'], + return makeConflictResponse('PUTFILE', acctok['userid'], retrievedlock, lock, 'NA', + acctok['endpoint'], acctok['filename'], reason + ', conflict copy created') From e398f5c8a9731e2eecb8d4f9aee753d7ed48414a Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 15 Sep 2022 15:35:39 +0200 Subject: [PATCH 006/325] Bridged apps: reverted IP resolution logic, we only rely on custom header --- src/bridge/__init__.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 51fd6312..6deda19e 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -6,7 +6,6 @@ import sys import time -import socket import traceback import threading import atexit @@ -88,12 +87,9 @@ def loadplugin(cls, appname, appurl, appinturl, apikey): cls.plugins[p].log = cls.log cls.plugins[p].sslverify = cls.sslverify cls.plugins[p].disablezip = cls.disablezip - addrinfo = socket.getaddrinfo(urlparse.urlparse(appinturl).netloc.split(':')[0], None, proto=socket.IPPROTO_TCP) - cls.plugins[p].remoteaddrs = list({addr[-1][0] for addr in addrinfo}) cls.plugins[p].appname = appname cls.plugins[p].init(appurl, appinturl, apikey) - cls.log.info('msg="Imported plugin for application" app="%s" plugin="%s" authorizedfrom="%s"' % - (p, cls.plugins[p], cls.plugins[p].remoteaddrs)) + cls.log.info('msg="Imported plugin for application" app="%s" plugin="%s"' % (p, cls.plugins[p])) except Exception as e: cls.log.info('msg="Failed to initialize plugin" app="%s" URL="%s" exception="%s"' % (p, appinturl, e)) @@ -117,14 +113,6 @@ def isextsupported(fileext): return fileext.lower() in set(BRIDGE_EXT_PLUGINS.keys()) -def _getappnamebyaddr(remoteaddr): - '''Return the appname of a (supported) app given its remote IP address''' - for p in WB.plugins.values(): - if remoteaddr in p.remoteaddrs: - return p.appname - raise ValueError - - def _validateappname(appname): '''Return the plugin's appname if one of the registered plugins matches (case-insensitive) the given appname''' for p in WB.plugins.values(): @@ -232,13 +220,8 @@ def appsave(docid): isclose = flask.request.args.get('close') == 'true' # ensure a save request comes from known/registered applications: - # this is done via a specific header, falling back to reverse IP resolution - # (note that the latter fails with apps deployed in k8s clusters) - # both functions raise ValueError if not found - if BRIDGED_APP_HEADER in flask.request.headers: - appname = _validateappname(flask.request.headers[BRIDGED_APP_HEADER]) - else: - appname = _getappnamebyaddr(flask.request.remote_addr) + # this is done via a specific header + appname = _validateappname(flask.request.headers[BRIDGED_APP_HEADER]) WB.log.info('msg="BridgeSave: requested action" isclose="%s" docid="%s" app="%s" wopisrc="%s" token="%s"' % (isclose, docid, appname, wopisrc, acctok[-20:])) except KeyError as e: @@ -246,7 +229,7 @@ def appsave(docid): (flask.request.remote_addr, flask.request.headers, flask.request.args, e)) return wopic.jsonify('Missing metadata, could not save. %s' % RECOVER_MSG), http.client.BAD_REQUEST except ValueError as e: - WB.log.error('msg="BridgeSave: unknown application" address="%s" appHeader="%s" args="%s"' % + WB.log.error('msg="BridgeSave: unknown application" address="%s" appheader="%s" args="%s"' % (flask.request.remote_addr, flask.request.headers.get(BRIDGED_APP_HEADER), flask.request.args)) # temporary override appname = 'CodiMD' From 1b5681dd9aeca4d084ff99235b4ad6705e0154aa Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 15 Sep 2022 14:57:39 +0200 Subject: [PATCH 007/325] Added logic to keep track of conflicted sessions These are the sessions that got an outstanding conflict. A dedicated endpoint was also added to query the list of still outstanding + resolved sessions. --- src/core/wopi.py | 20 ++++++-------------- src/core/wopiutils.py | 20 +++++++++++++++++++- src/wopiserver.py | 20 ++++++++++++++++++++ tools/wopilistconflicts.sh | 2 ++ 4 files changed, 47 insertions(+), 15 deletions(-) create mode 100755 tools/wopilistconflicts.sh diff --git a/src/core/wopi.py b/src/core/wopi.py index f26aa9e3..799feb97 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -221,8 +221,6 @@ def setLock(fileid, reqheaders, acctok): # LOCK or REFRESH_LOCK: atomically set the lock to the given one, including the expiration time, # and return conflict response if the file was already locked st.setlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(lock)) - log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s"' % - (op.title(), fn, flask.request.args['access_token'][-20:], flask.request.headers.get('X-WOPI-SessionId'), lock)) # on first lock, set an xattr with the current time for later conflicts checking try: @@ -233,17 +231,15 @@ def setLock(fileid, reqheaders, acctok): log.warning('msg="Unable to set lastwritetime xattr" lockop="%s" user="%s" filename="%s" token="%s" reason="%s"' % (op.title(), acctok['userid'][-20:], fn, flask.request.args['access_token'][-20:], e)) # also, keep track of files that have been opened for write: this is for statistical purposes only - # (cf. the GetLock WOPI call and the /wopi/cbox/open/list action) + # (cf. the GetLock WOPI call and the /wopi/iop/open/list action) if fn not in srv.openfiles: srv.openfiles[fn] = (time.asctime(), set([acctok['username']])) else: # the file was already opened but without lock: this happens on new files (cf. editnew action), just log log.info('msg="First lock for new file" lockop="%s" user="%s" filename="%s" token="%s"' % (op.title(), acctok['userid'][-20:], fn, flask.request.args['access_token'][-20:])) - resp = flask.Response() - resp.status_code = http.client.OK - resp.headers['X-WOPI-ItemVersion'] = 'v%s' % statInfo['etag'] - return resp + + return utils.makeLockSuccessResponse(op, fn, lock, 'v%s' % statInfo['etag']) except IOError as e: if common.EXCL_ERROR in str(e): @@ -257,21 +253,17 @@ def setLock(fileid, reqheaders, acctok): return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, acctok['endpoint'], fn, 'The file is locked by %s' % (lockHolder if lockHolder != 'wopi' else 'another online editor')) + # else it's our own lock, refresh it and return try: st.refreshlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(lock)) - log.info('msg="Successfully refreshed" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s"' % - (op.title(), fn, flask.request.args['access_token'][-20:], flask.request.headers.get('X-WOPI-SessionId'), lock)) - # else we don't need to refresh it again - resp = flask.Response() - resp.status_code = http.client.OK - resp.headers['X-WOPI-ItemVersion'] = 'v%s' % statInfo['etag'] - return resp + return utils.makeLockSuccessResponse(op, fn, lock, 'v%s' % statInfo['etag']) except IOError as rle: # this is unexpected now log.error('msg="Failed to refresh lock" lockop="%s" filename="%s" token="%s" lock="%s" error="%s"' % (op.title(), fn, flask.request.args['access_token'][-20:], lock, rle)) + # any other error is raised log.error('msg="Unable to store WOPI lock" lockop="%s" filename="%s" token="%s" lock="%s" error="%s"' % (op.title(), fn, flask.request.args['access_token'][-20:], lock, e)) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 99633ad1..b436732d 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -346,14 +346,32 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, endpoint savetime = int(savetime) else: savetime = 0 + session = flask.request.headers.get('X-WOPI-SessionId') + if session: + srv.conflictsessions['pending'].add(session) log.warning('msg="Returning conflict" lockop="%s" user="%s" filename="%s" token="%s" sessionId="%s" lock="%s" ' 'oldlock="%s" retrievedlock="%s" fileage="%s" reason="%s"' % (operation.title(), user, filename, flask.request.args['access_token'][-20:], - flask.request.headers.get('X-WOPI-SessionId'), lock, oldlock, retrievedlock, time.time() - savetime, + session, lock, oldlock, retrievedlock, time.time() - savetime, (reason['message'] if reason else 'NA'))) return resp +def makeLockSuccessResponse(operation, filename, lock, version): + '''Generates and logs an HTTP 200 response with appropriate headers for Lock/RefreshLock operations''' + session = flask.request.headers.get('X-WOPI-SessionId') + if session in srv.conflictsessions['pending']: + srv.conflictsessions['pending'].remove(session) + srv.conflictsessions['resolved'].add(session) + + log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s"' % + (operation.title(), filename, flask.request.args['access_token'][-20:], session, lock)) + resp = flask.Response() + resp.status_code = http.client.OK + resp.headers['X-WOPI-ItemVersion'] = version + return resp + + def storeWopiFile(acctok, retrievedlock, xakey, targetname=''): '''Saves a file from an HTTP request to the given target filename (defaulting to the access token's one), and stores the save time as an xattr. Throws IOError in case of any failure''' diff --git a/src/wopiserver.py b/src/wopiserver.py index bf6ab31a..d420741b 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -74,6 +74,9 @@ class Wopi: } log = utils.JsonLogger(app.logger) openfiles = {} + # sets of sessions for which a lock conflict is outstanding or resolved + conflictsessions = {'pending': set(), 'resolved': set()} + @classmethod def init(cls): @@ -381,6 +384,23 @@ def iopGetOpenFiles(): return flask.Response(json.dumps(jlist), mimetype='application/json') +@Wopi.app.route("/wopi/iop/conflicts", methods=['GET']) +def iopGetConflicts(): + '''Returns a list of all currently outstanding and resolved conflicted sessions, for operators only. + This call is protected by the same shared secret as the /wopi/iop/openinapp call.''' + req = flask.request + if req.headers.get('Authorization') != 'Bearer ' + Wopi.iopsecret: + Wopi.log.warning('msg="iopGetConflicts: unauthorized access attempt, missing authorization token" ' + 'client="%s"' % req.remote_addr) + return UNAUTHORIZED + # dump the current sets in JSON format + jlist = {} + for l in Wopi.conflictsessions.keys(): + jlist[l] = list(Wopi.conflictsessions[l]) + Wopi.log.info('msg="iopGetConflicts: returning outstanding/resolved conflicted sessions" client="%s"' % req.remote_addr) + return flask.Response(json.dumps(jlist), mimetype='application/json') + + @Wopi.app.route("/wopi/iop/test", methods=['GET']) def iopWopiTest(): '''Returns a WOPI_URL and a WOPI_TOKEN values suitable as input for the WOPI validator test suite. diff --git a/tools/wopilistconflicts.sh b/tools/wopilistconflicts.sh new file mode 100755 index 00000000..5c6d2957 --- /dev/null +++ b/tools/wopilistconflicts.sh @@ -0,0 +1,2 @@ +curl --insecure --header "Authorization: Bearer "`cat /etc/wopi/iopsecret` https://`hostname`:8443/wopi/iop/conflicts +echo From 9351dff2a3a635179d067032d6be01c37e4916bd Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 16 Sep 2022 15:30:18 +0200 Subject: [PATCH 008/325] Added optional timeout for xroot operations --- src/core/xrootiface.py | 61 +++++++++++++++++++++++++----------------- wopiserver.conf | 3 +++ 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 23542793..1d2c78cb 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -25,6 +25,24 @@ defaultstorage = None endpointoverride = None homepath = None +timeout = None + +def init(inconfig, inlog): + '''Init module-level variables''' + global config # pylint: disable=global-statement + global log # pylint: disable=global-statement + global endpointoverride # pylint: disable=global-statement + global defaultstorage # pylint: disable=global-statement + global homepath # pylint: disable=global-statement + global timeout # pylint: disable=global-statement + common.config = config = inconfig + log = inlog + endpointoverride = config.get('xroot', 'endpointoverride', fallback='') + defaultstorage = config.get('xroot', 'storageserver') + homepath = config.get('xroot', 'storagehomepath', fallback='') + timeout = int(config.get('xroot', 'timeout', fallback='10')) + # prepare the xroot client for the default storageserver + _getxrdfor(defaultstorage) def _getxrdfor(endpoint): @@ -77,8 +95,11 @@ def _xrootcmd(endpoint, cmd, subcmd, userid, args): url = _geturlfor(endpoint) + '//proc/user/' + _eosargs(userid) + '&mgm.cmd=' + cmd + \ ('&mgm.subcmd=' + subcmd if subcmd else '') + '&' + args tstart = time.time() - rc, _ = f.open(url, OpenFlags.READ) + rc, _ = f.open(url, OpenFlags.READ, timeout=timeout) tend = time.time() + if not f.is_open(): + log.error('msg="Timeout with xroot" cmd="%s" subcmd="%s" args="%s"' % (cmd, subcmd, args)) + raise IOError('Timeout executing %s' % cmd) res = b''.join(f.readlines()).decode().split('&') if len(res) == 3: # we may only just get stdout: in that case, assume it's all OK rc = res[2].strip('\n') @@ -110,22 +131,6 @@ def _getfilepath(filepath, encodeamp=False): return homepath + (filepath if not encodeamp else filepath.replace('&', '#AND#')) -def init(inconfig, inlog): - '''Init module-level variables''' - global config # pylint: disable=global-statement - global log # pylint: disable=global-statement - global endpointoverride # pylint: disable=global-statement - global defaultstorage # pylint: disable=global-statement - global homepath # pylint: disable=global-statement - common.config = config = inconfig - log = inlog - endpointoverride = config.get('xroot', 'endpointoverride', fallback='') - defaultstorage = config.get('xroot', 'storageserver') - homepath = config.get('xroot', 'storagehomepath', fallback='') - # prepare the xroot client for the default storageserver - _getxrdfor(defaultstorage) - - def getuseridfromcreds(_token, wopiuser): '''Maps a Reva token and wopiuser to the credentials to be used to access the storage. For the xrootd case, we have to resolve the username to uid:gid''' @@ -137,7 +142,7 @@ def stat(endpoint, filepath, userid): '''Stat a file via xroot on behalf of the given userid, and returns (size, mtime). Uses the default xroot API.''' filepath = _getfilepath(filepath, encodeamp=True) tstart = time.time() - rc, statInfo = _getxrdfor(endpoint).stat(filepath + _eosargs(userid)) + rc, statInfo = _getxrdfor(endpoint).stat(filepath + _eosargs(userid), timeout=timeout) tend = time.time() log.info('msg="Invoked stat" filepath="%s" elapsedTimems="%.1f"' % (filepath, (tend - tstart) * 1000)) if not statInfo: @@ -204,17 +209,19 @@ def statx(endpoint, fileref, userid, versioninv=1): # also, use the owner's as opposed to the user's credentials to bypass any restriction (e.g. with single-share files) verFolder = os.path.dirname(filepath) + os.path.sep + EOSVERSIONPREFIX + os.path.basename(filepath) ownerarg = _eosargs(statxdata['uid'] + ':' + statxdata['gid']) - rcv, infov = _getxrdfor(endpoint).query(QueryCode.OPAQUEFILE, _getfilepath(verFolder) + ownerarg + '&mgm.pcmd=stat') + rcv, infov = _getxrdfor(endpoint).query(QueryCode.OPAQUEFILE, _getfilepath(verFolder) + ownerarg + '&mgm.pcmd=stat', + timeout=timeout) tend = time.time() infov = infov.decode() try: if OK_MSG not in str(rcv) or 'retc=2' in infov: # the version folder does not exist: create it (on behalf of the owner) as it is done in Reva - rcmkdir = _getxrdfor(endpoint).mkdir(_getfilepath(verFolder) + ownerarg, MkDirFlags.MAKEPATH) + rcmkdir = _getxrdfor(endpoint).mkdir(_getfilepath(verFolder) + ownerarg, MkDirFlags.MAKEPATH, timeout=timeout) if OK_MSG not in str(rcmkdir): raise IOError(rcmkdir) log.debug('msg="Invoked mkdir on version folder" filepath="%s"' % _getfilepath(verFolder)) - rcv, infov = _getxrdfor(endpoint).query(QueryCode.OPAQUEFILE, _getfilepath(verFolder) + ownerarg + '&mgm.pcmd=stat') + rcv, infov = _getxrdfor(endpoint).query(QueryCode.OPAQUEFILE, _getfilepath(verFolder) + ownerarg + '&mgm.pcmd=stat', + timeout=timeout) tend = time.time() infov = infov.decode() if OK_MSG not in str(rcv) or 'retc=' in infov: @@ -320,10 +327,13 @@ def readfile(endpoint, filepath, userid, _lockid): '''Read a file via xroot on behalf of the given userid. Note that the function is a generator, managed by Flask.''' log.debug('msg="Invoking readFile" filepath="%s"' % filepath) with XrdClient.File() as f: - fileurl = _geturlfor(endpoint) + '/' + homepath + filepath + _eosargs(userid) tstart = time.time() - rc, _ = f.open(fileurl, OpenFlags.READ) + rc, _ = f.open(_geturlfor(endpoint) + '/' + homepath + filepath + _eosargs(userid), + OpenFlags.READ, timeout=timeout) tend = time.time() + if not f.is_open(): + log.error('msg="Timeout with xroot" op="read" filepath="%s"' % filepath) + raise IOError('Timeout opening file for read') if not rc.ok: # the file could not be opened: check the case of ENOENT and log it as info to keep the logs cleaner if common.ENOENT_MSG in rc.message: @@ -354,8 +364,11 @@ def writefile(endpoint, filepath, userid, content, _lockid, islock=False): f = XrdClient.File() tstart = time.time() rc, _ = f.open(_geturlfor(endpoint) + '/' + homepath + filepath + _eosargs(userid, not islock, size), - OpenFlags.NEW if islock else OpenFlags.DELETE) + OpenFlags.NEW if islock else OpenFlags.DELETE, timeout=timeout) tend = time.time() + if not f.is_open(): + log.error('msg="Timeout with xroot" op="write" filepath="%s"' % filepath) + raise IOError('Timeout opening file for write') if not rc.ok: if islock and 'File exists' in rc.message: # racing against an existing file diff --git a/wopiserver.conf b/wopiserver.conf index 68d0fc30..d0d49d0c 100644 --- a/wopiserver.conf +++ b/wopiserver.conf @@ -158,6 +158,9 @@ chunksize = 4194304 # this is not used and storagehomepath is empty. #storagehomepath = /your/top/storage/path +# Optional timeout value [seconds] applied to all xroot requests. +#timeout = 10 + [local] # Location of the folder or mount point used as local storage From 8bd8b59634ea11e6124af6bf99cc6e1747fe02ac Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 17 Sep 2022 18:49:05 +0200 Subject: [PATCH 009/325] Better implementation of the disablemswriteodf feature Now the openInApp returns the view URL as opposed to the edit URL for ODF files opened by Microsoft Office --- src/core/wopi.py | 8 ++++---- src/core/wopiutils.py | 2 +- src/wopiserver.py | 16 ++++++++-------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 799feb97..e3d7c8e6 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -402,10 +402,10 @@ def putRelative(fileid, reqheaders, acctok): log.info('msg="PutRelative: generating new access token" user="%s" filename="%s" ' 'mode="ViewMode.READ_WRITE" friendlyname="%s"' % (acctok['userid'][-20:], targetName, acctok['username'])) - inode, newacctok = utils.generateAccessToken(acctok['userid'], targetName, utils.ViewMode.READ_WRITE, - (acctok['username'], acctok['wopiuser']), - acctok['folderurl'], acctok['endpoint'], - (acctok['appname'], acctok['appediturl'], acctok['appviewurl'])) + inode, newacctok, _ = utils.generateAccessToken(acctok['userid'], targetName, utils.ViewMode.READ_WRITE, + (acctok['username'], acctok['wopiuser']), + acctok['folderurl'], acctok['endpoint'], + (acctok['appname'], acctok['appediturl'], acctok['appviewurl'])) # prepare and send the response as JSON putrelmd = {} putrelmd['Name'] = os.path.basename(targetName) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index b436732d..e434c276 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -208,7 +208,7 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app statinfo['filepath'], statinfo['inode'], statinfo['mtime'], folderurl, appname, exptime, acctok[-20:])) # return the inode == fileid, the filepath and the access token - return statinfo['inode'], acctok + return statinfo['inode'], acctok, viewmode def encodeLock(lock): diff --git a/src/wopiserver.py b/src/wopiserver.py index d420741b..07d415d6 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -327,8 +327,8 @@ def iopOpenInApp(): # this happens in hybrid deployments with xrootd as storage interface: # in this case we override the wopiuser with the resolved uid:gid wopiuser = userid - inode, acctok = utils.generateAccessToken(userid, fileid, viewmode, (username, wopiuser), folderurl, endpoint, - (appname, appurl, appviewurl)) + inode, acctok, vm = utils.generateAccessToken(userid, fileid, viewmode, (username, wopiuser), folderurl, endpoint, + (appname, appurl, appviewurl)) except IOError as e: Wopi.log.info('msg="iopOpenInApp: remote error on generating token" client="%s" user="%s" ' 'friendlyname="%s" mode="%s" endpoint="%s" reason="%s"' % @@ -342,7 +342,7 @@ def iopOpenInApp(): except bridge.FailedOpen as foe: return foe.msg, foe.statuscode else: - res['app-url'] = appurl if viewmode == utils.ViewMode.READ_WRITE else appviewurl + res['app-url'] = appurl if vm == utils.ViewMode.READ_WRITE else appviewurl res['app-url'] += '%sWOPISrc=%s' % ('&' if '?' in res['app-url'] else '?', utils.generateWopiSrc(inode, appname == Wopi.proxiedappname)) res['form-parameters'] = {'access_token': acctok} @@ -423,9 +423,9 @@ def iopWopiTest(): return 'Missing arguments', http.client.BAD_REQUEST if Wopi.useHttps: return 'WOPI validator not supported in https mode', http.client.BAD_REQUEST - inode, acctok = utils.generateAccessToken(usertoken, filepath, utils.ViewMode.READ_WRITE, ('test', usertoken), - 'http://folderurlfortestonly/', endpoint, - ('WOPI validator', 'http://fortestonly/', 'http://fortestonly/')) + inode, acctok, _ = utils.generateAccessToken(usertoken, filepath, utils.ViewMode.READ_WRITE, ('test', usertoken), + 'http://folderurlfortestonly/', endpoint, + ('WOPI validator', 'http://fortestonly/', 'http://fortestonly/')) Wopi.log.info('msg="iopWopiTest: preparing test via WOPI validator" client="%s"' % req.remote_addr) return '-e WOPI_URL=http://localhost:%d/wopi/files/%s -e WOPI_TOKEN=%s' % (Wopi.port, inode, acctok) @@ -578,8 +578,8 @@ def cboxOpen_deprecated(): toproxy = req.args.get('proxy', 'false') == 'true' and filename[-1] == 'x' # if requested, only proxy OOXML files try: # here we set wopiuser = userid (i.e. uid:gid) as that's well known to be consistent over time - inode, acctok = utils.generateAccessToken(userid, filename, viewmode, (username, userid), - folderurl, endpoint, (Wopi.proxiedappname if toproxy else '', '', '')) + inode, acctok, _ = utils.generateAccessToken(userid, filename, viewmode, (username, userid), + folderurl, endpoint, (Wopi.proxiedappname if toproxy else '', '', '')) except IOError as e: Wopi.log.warning('msg="cboxOpen: remote error on generating token" client="%s" user="%s" ' 'friendlyname="%s" mode="%s" endpoint="%s" reason="%s"' % From 68eb4226a6041f13ca6a7218158f49c89aadce67 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 19 Sep 2022 09:05:11 +0200 Subject: [PATCH 010/325] Added Javi as code owner to help with reviews --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 724c81e3..bf9b1641 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @glpatcern +* @glpatcern @javfg From 63f55e78922952b27dbe799f97af969ecf6c7fce Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 21 Sep 2022 09:14:38 +0200 Subject: [PATCH 011/325] Also add a counter to the conflicted sessions stats --- src/core/wopiutils.py | 9 ++++++--- src/wopiserver.py | 7 ++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index e434c276..a85f1d2e 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -348,7 +348,10 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, endpoint savetime = 0 session = flask.request.headers.get('X-WOPI-SessionId') if session: - srv.conflictsessions['pending'].add(session) + if session in srv.conflictsessions['pending']: + srv.conflictsessions['pending'][session] += 1 + else: + srv.conflictsessions['pending'][session] = 1 log.warning('msg="Returning conflict" lockop="%s" user="%s" filename="%s" token="%s" sessionId="%s" lock="%s" ' 'oldlock="%s" retrievedlock="%s" fileage="%s" reason="%s"' % (operation.title(), user, filename, flask.request.args['access_token'][-20:], @@ -361,8 +364,8 @@ def makeLockSuccessResponse(operation, filename, lock, version): '''Generates and logs an HTTP 200 response with appropriate headers for Lock/RefreshLock operations''' session = flask.request.headers.get('X-WOPI-SessionId') if session in srv.conflictsessions['pending']: - srv.conflictsessions['pending'].remove(session) - srv.conflictsessions['resolved'].add(session) + counter = srv.conflictsessions['pending'].pop(session) + srv.conflictsessions['resolved'][session] = counter log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s"' % (operation.title(), filename, flask.request.args['access_token'][-20:], session, lock)) diff --git a/src/wopiserver.py b/src/wopiserver.py index 07d415d6..0f3e18fc 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -75,7 +75,7 @@ class Wopi: log = utils.JsonLogger(app.logger) openfiles = {} # sets of sessions for which a lock conflict is outstanding or resolved - conflictsessions = {'pending': set(), 'resolved': set()} + conflictsessions = {'pending': {}, 'resolved': {}} @classmethod @@ -394,11 +394,8 @@ def iopGetConflicts(): 'client="%s"' % req.remote_addr) return UNAUTHORIZED # dump the current sets in JSON format - jlist = {} - for l in Wopi.conflictsessions.keys(): - jlist[l] = list(Wopi.conflictsessions[l]) Wopi.log.info('msg="iopGetConflicts: returning outstanding/resolved conflicted sessions" client="%s"' % req.remote_addr) - return flask.Response(json.dumps(jlist), mimetype='application/json') + return flask.Response(json.dumps(Wopi.conflictsessions), mimetype='application/json') @Wopi.app.route("/wopi/iop/test", methods=['GET']) From 4f79013d4727a513c2a65346a061baad90737aaa Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sun, 4 Sep 2022 17:46:35 +0200 Subject: [PATCH 012/325] Enabled support for preview mode --- src/bridge/__init__.py | 4 ++-- src/bridge/codimd.py | 13 +++++++------ src/bridge/etherpad.py | 8 ++++---- src/core/wopiutils.py | 9 +++++++-- src/wopiserver.py | 6 ++++-- tools/wopiopen.py | 2 +- 6 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 6deda19e..dcfce750 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -130,7 +130,7 @@ def _gendocid(wopisrc): # The Bridge endpoints start here ############################################################################################################# -def appopen(wopisrc, acctok, appname): +def appopen(wopisrc, acctok, appname, viewmode): '''Open a doc by contacting the provided WOPISrc with the given access_token. Returns a (app-url, params{}) pair if successful, raises a FailedOpen exception otherwise''' wopisrc = urlparse.unquote_plus(wopisrc) @@ -200,7 +200,7 @@ def appopen(wopisrc, acctok, appname): # user has no write privileges, just fetch the document and push it to the app on a random docid wopilock = app.loadfromstorage(filemd, wopisrc, acctok, None) - redirurl = app.getredirecturl(filemd['UserCanWrite'], wopisrc, acctok, wopilock['doc'][1:], + redirurl = app.getredirecturl(viewmode, wopisrc, acctok, wopilock['doc'][1:], urlparse.quote_plus(filemd['UserFriendlyName'])) except app.AppFailure as e: # this can be raised by loadfromstorage or getredirecturl diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 8b57c803..25969a79 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -16,7 +16,7 @@ import http.client import requests import bridge.wopiclient as wopic - +import core.wopiutils as utils TOOLARGE = 'File is too large to be edited in CodiMD. Please reduce its size with a regular text editor and try again.' @@ -56,11 +56,12 @@ def init(_appurl, _appinturl, _apikey): raise AppFailure -def getredirecturl(isreadwrite, wopisrc, acctok, docid, displayname): +def getredirecturl(viewmode, wopisrc, acctok, docid, displayname): '''Return a valid URL to the app for the given WOPI context''' - if isreadwrite: - return '%s/%s?wopiSrc=%s&accessToken=%s&displayName=%s' % \ - (appexturl, docid, urlparse.quote_plus(wopisrc), acctok, displayname) + if viewmode in (utils.ViewMode.READ_WRITE, utils.ViewMode.PREVIEW): + return '%s/%s?%s&wopiSrc=%s&accessToken=%s&apiKey=%s&displayName=%s' % \ + (appexturl, docid, ('view' if viewmode == utils.ViewMode.PREVIEW else 'both'), + urlparse.quote_plus(wopisrc), acctok, apikey, displayname) # read-only mode: first check if we have a CodiMD redirection res = requests.head(appurl + '/' + docid, @@ -236,7 +237,7 @@ def savetostorage(wopisrc, acctok, isclose, wopilock, onlyfetch=False): '''Copy document from CodiMD back to storage''' # get document from CodiMD try: - log.info('msg="Fetching file from CodiMD" isclose="%s" url="%s" token="%s"' % + log.info('msg="Fetching file from CodiMD" isclose="%s" appurl="%s" token="%s"' % (isclose, appurl + wopilock['doc'], acctok[-20:])) mddoc = _fetchfromcodimd(wopilock, acctok) if onlyfetch: diff --git a/src/bridge/etherpad.py b/src/bridge/etherpad.py index b2a8a51f..49403a2b 100644 --- a/src/bridge/etherpad.py +++ b/src/bridge/etherpad.py @@ -13,7 +13,7 @@ import urllib.parse as urlparse import requests import bridge.wopiclient as wopic - +import core.wopiutils as utils # initialized by the main class or by the init method appurl = None @@ -65,7 +65,7 @@ def _apicall(method, params, data=None, acctok=None, raiseonnonzerocode=True): return res -def getredirecturl(isreadwrite, wopisrc, acctok, docid, displayname): +def getredirecturl(viewmode, wopisrc, acctok, docid, displayname): '''Return a valid URL to the app for the given WOPI context''' # pass to Etherpad the required metadata for the save webhook try: @@ -82,11 +82,11 @@ def getredirecturl(isreadwrite, wopisrc, acctok, docid, displayname): log.error('msg="Exception raised attempting to connect to Etherpad" method="setEFSSMetadata" exception="%s"' % e) raise AppFailure - if not isreadwrite: + if viewmode in (utils.ViewMode.READ_ONLY, utils.ViewMode.VIEW_ONLY): # for read-only mode generate a read-only link res = _apicall('getReadOnlyID', {'padID': docid}, acctok=acctok) return appexturl + '/p/%s?userName=%s' % (res['data']['readOnlyID'], displayname) - # return the URL to the pad + # return the URL to the pad (TODO if viewmode is PREVIEW) return appexturl + '/p/%s?userName=%s' % (docid, displayname) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index a85f1d2e..575b1049 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -43,8 +43,10 @@ class ViewMode(Enum): VIEW_ONLY = "VIEW_MODE_VIEW_ONLY" # The file can be downloaded READ_ONLY = "VIEW_MODE_READ_ONLY" - # The file can be downloaded and updated + # The file can be downloaded and updated, and the app should be shown in edit mode READ_WRITE = "VIEW_MODE_READ_WRITE" + # The file can be downloaded and updated, and the app should be shown in preview mode + PREVIEW = "VIEW_MODE_PREVIEW" class JsonLogger: @@ -191,8 +193,11 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app log.critical('msg="No app URLs registered for the given file type" fileext="%s" mimetypescount="%d"' % (fext, len(endpoints) if endpoints else 0)) raise IOError + if viewmode == ViewMode.PREVIEW: + # preview mode assumes read/write privileges for the acctok + viewmode = ViewMode.READ_WRITE if srv.config.get('general', 'disablemswriteodf', fallback='False').upper() == 'TRUE' and \ - fext in srv.codetypes and appname != 'Collabora' and appname != '' and viewmode == ViewMode.READ_WRITE: + fext in srv.codetypes and appname not in ('Collabora', '') and viewmode == ViewMode.READ_WRITE: # we're opening an ODF file and the app is not Collabora (the last check is needed because the legacy endpoint # does not set appname when the app is not proxied, so we optimistically assume it's Collabora and let it go) log.info('msg="Forcing read-only access to ODF file" filename="%s"' % statinfo['filepath']) diff --git a/src/wopiserver.py b/src/wopiserver.py index 0f3e18fc..426a139d 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -338,10 +338,12 @@ def iopOpenInApp(): res = {} if bridge.issupported(appname): try: - res['app-url'], res['form-parameters'] = bridge.appopen(utils.generateWopiSrc(inode), acctok, appname) + res['app-url'], res['form-parameters'] = bridge.appopen(utils.generateWopiSrc(inode), acctok, appname, viewmode) except bridge.FailedOpen as foe: return foe.msg, foe.statuscode else: + # the base app URL is the editor in READ_WRITE mode, and the viewer in READ_ONLY or PREVIEW mode + # as the known WOPI applications all support switching from preview to edit mode res['app-url'] = appurl if vm == utils.ViewMode.READ_WRITE else appviewurl res['app-url'] += '%sWOPISrc=%s' % ('&' if '?' in res['app-url'] else '?', utils.generateWopiSrc(inode, appname == Wopi.proxiedappname)) @@ -586,7 +588,7 @@ def cboxOpen_deprecated(): # call the bridgeOpen right away, to not expose the WOPI URL to the user (it might be behind firewall) try: appurl, _ = bridge.appopen(utils.generateWopiSrc(inode), acctok, - bridge.BRIDGE_EXT_PLUGINS[os.path.splitext(filename)[1][1:]]) + bridge.BRIDGE_EXT_PLUGINS[os.path.splitext(filename)[1][1:]], viewmode) Wopi.log.debug('msg="cboxOpen: returning bridged app" URL="%s"' % appurl[appurl.rfind('/'):]) return appurl[appurl.rfind('/'):] # return the payload as the appurl is already known via discovery except bridge.FailedOpen as foe: diff --git a/tools/wopiopen.py b/tools/wopiopen.py index 434695ea..46038b5e 100755 --- a/tools/wopiopen.py +++ b/tools/wopiopen.py @@ -20,7 +20,7 @@ def usage(exitcode): '''Prints usage''' print('Usage : ' + sys.argv[0] + ' -a|--appname -u|--appurl [-i|--appinturl ] -k|--apikey ' - '[-s|--storage ] [-v|--viewmode VIEW_ONLY|READ_ONLY|READ_WRITE] [-x|--x-access-token ] ') + '[-s|--storage ] [-v|--viewmode VIEW_ONLY|READ_ONLY|READ_WRITE|PREVIEW] [-x|--x-access-token ] ') sys.exit(exitcode) From 375749b38fe338695e32f397301d0d6f3fe71b9e Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 21 Sep 2022 11:45:46 +0200 Subject: [PATCH 013/325] Implemented cleaner way to generate URLs Co-authored-by: Javier Ferrer --- src/bridge/codimd.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 25969a79..52924be6 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -59,9 +59,14 @@ def init(_appurl, _appinturl, _apikey): def getredirecturl(viewmode, wopisrc, acctok, docid, displayname): '''Return a valid URL to the app for the given WOPI context''' if viewmode in (utils.ViewMode.READ_WRITE, utils.ViewMode.PREVIEW): - return '%s/%s?%s&wopiSrc=%s&accessToken=%s&apiKey=%s&displayName=%s' % \ - (appexturl, docid, ('view' if viewmode == utils.ViewMode.PREVIEW else 'both'), - urlparse.quote_plus(wopisrc), acctok, apikey, displayname) + mode = 'view' if viewmode == utils.ViewMode.PREVIEW else 'both' + params = { + 'wopiSrc': wopisrc, + 'accessToken': acctok, + 'apiKey': apikey, + 'displayName': displayname, + } + return f'{appexturl}/{docid}?{mode}&{urlparse.urlencode(params)}' # read-only mode: first check if we have a CodiMD redirection res = requests.head(appurl + '/' + docid, @@ -71,7 +76,7 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, displayname): # we used to redirect to publish mode or normal view to quickly jump in slide mode depending on the content, # but this was based on a bad side effect - here it would require to add: # ('/publish' if not _isslides(content) else '') before the '?' - return '%s/%s/publish' % (appexturl, docid) + return f'{appexturl}/{docid}/publish' # Cloud storage to CodiMD From 7d3c3c0977a117405e9f451feba57d95232ede13 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Wed, 21 Sep 2022 23:26:43 +0200 Subject: [PATCH 014/325] fix UnLockAndRelock and length of resourceIDs --- src/core/cs3iface.py | 2 +- src/core/wopi.py | 4 ++++ src/core/wopiutils.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index 57d0b27b..ca3064ef 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -182,7 +182,7 @@ def setlock(endpoint, filepath, userid, appname, value): expiration={'seconds': int(time.time() + ctx['lockexpiration'])}) req = cs3sp.SetLockRequest(ref=reference, lock=lock) res = ctx['cs3gw'].SetLock(request=req, metadata=[('x-access-token', userid)]) - if res.status.code == cs3code.CODE_FAILED_PRECONDITION: + if res.status.code in [cs3code.CODE_FAILED_PRECONDITION, cs3code.CODE_ABORTED]: log.info('msg="Invoked setlock on an already locked entity" filepath="%s" appname="%s" trace="%s" reason="%s"' % (filepath, appname, res.status.trace, res.status.message.replace('"', "'"))) raise IOError(common.EXCL_ERROR) diff --git a/src/core/wopi.py b/src/core/wopi.py index e3d7c8e6..b7c7d493 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -218,6 +218,10 @@ def setLock(fileid, reqheaders, acctok): (op.title(), fn, flask.request.args['access_token'][-20:], e)) try: + # If LOCK and oldLock provided this is an UnLockAndRelock operation + if op == 'LOCK' and oldLock: + st.unlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(oldLock)) + # LOCK or REFRESH_LOCK: atomically set the lock to the given one, including the expiration time, # and return conflict response if the file was already locked st.setlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(lock)) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 575b1049..c09545d3 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -138,7 +138,7 @@ def generateWopiSrc(fileid, proxy=False): if not proxy or not srv.wopiproxy: return url_quote_plus('%s/wopi/files/%s' % (srv.wopiurl, fileid)).replace('-', '%2D') # proxy the WOPI request through an external WOPI proxy service, but only if it was not already proxied - if len(fileid) < 50: # heuristically, proxied fileids are (much) longer than that + if len(fileid) < 90: # heuristically, proxied fileids are (much) longer than that log.debug('msg="Generating proxied fileid" fileid="%s" proxy="%s"' % (fileid, srv.wopiproxy)) fileid = jwt.encode({'u': srv.wopiurl + '/wopi/files/', 'f': fileid}, srv.wopiproxykey, algorithm='HS256') else: From afe0d94caa0de12ed90e3dcb7756001a982cf4d3 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Fri, 30 Sep 2022 15:41:33 +0200 Subject: [PATCH 015/325] Use python-alpine docker base image --- wopiserver.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wopiserver.Dockerfile b/wopiserver.Dockerfile index 2ce1ad38..e22f4b47 100644 --- a/wopiserver.Dockerfile +++ b/wopiserver.Dockerfile @@ -2,7 +2,7 @@ # # Build: make docker or docker-compose -f wopiserver.yaml build --build-arg VERSION=`git describe | sed 's/^v//'` wopiserver -FROM python:3.10 +FROM python:3.10-alpine ARG VERSION=latest From f8545b0662f285471563d494cfe4a41eb4fc3f30 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 26 Sep 2022 10:07:06 +0200 Subject: [PATCH 016/325] Strengthened validation of access tokens --- src/core/wopiutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index c09545d3..0c241de8 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -100,9 +100,9 @@ def validateAndLogHeaders(op): # validate the access token try: acctok = jwt.decode(flask.request.args['access_token'], srv.wopisecret, algorithms=['HS256']) - if acctok['exp'] < time.time(): + if acctok['exp'] < time.time() or 'cs3org:wopiserver' not in acctok['iss']: raise jwt.exceptions.ExpiredSignatureError - except (jwt.exceptions.DecodeError, jwt.exceptions.ExpiredSignatureError) as e: + except (jwt.exceptions.DecodeError, jwt.exceptions.ExpiredSignatureError, KeyError) as e: log.info('msg="Expired or malformed token" client="%s" requestedUrl="%s" error="%s" token="%s"' % (flask.request.remote_addr, flask.request.base_url, str(type(e)) + ': ' + str(e), flask.request.args['access_token'])) return 'Invalid access token', http.client.UNAUTHORIZED From 2847ccbef8680f76f4876eb47d473ceec8d488b4 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 28 Sep 2022 10:43:27 +0200 Subject: [PATCH 017/325] CodiMD: support zmd files without automatic transition to md Cf. CERNBOX-2972: we want to support both "embedded markdown with pictures" and plain markdown, but we do not want to automatically switch from one to the other as that breaks URLs, shares, revisions, etc. --- src/bridge/__init__.py | 2 +- src/bridge/codimd.py | 8 ++++---- src/bridge/etherpad.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index dcfce750..46c91e1d 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -200,7 +200,7 @@ def appopen(wopisrc, acctok, appname, viewmode): # user has no write privileges, just fetch the document and push it to the app on a random docid wopilock = app.loadfromstorage(filemd, wopisrc, acctok, None) - redirurl = app.getredirecturl(viewmode, wopisrc, acctok, wopilock['doc'][1:], + redirurl = app.getredirecturl(viewmode, wopisrc, acctok, wopilock['doc'][1:], filemd['BaseFileName'], urlparse.quote_plus(filemd['UserFriendlyName'])) except app.AppFailure as e: # this can be raised by loadfromstorage or getredirecturl diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 52924be6..79babf4e 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -56,7 +56,7 @@ def init(_appurl, _appinturl, _apikey): raise AppFailure -def getredirecturl(viewmode, wopisrc, acctok, docid, displayname): +def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname): '''Return a valid URL to the app for the given WOPI context''' if viewmode in (utils.ViewMode.READ_WRITE, utils.ViewMode.PREVIEW): mode = 'view' if viewmode == utils.ViewMode.PREVIEW else 'both' @@ -65,6 +65,7 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, displayname): 'accessToken': acctok, 'apiKey': apikey, 'displayName': displayname, + 'allowEmbedding': os.path.splitext(filename)[1] == '.zmd', } return f'{appexturl}/{docid}?{mode}&{urlparse.urlencode(params)}' @@ -147,7 +148,7 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): wasbundle = os.path.splitext(filemd['BaseFileName'])[1] == '.zmd' # if it's a bundled file, unzip it and push the attachments in the appropriate folder - if wasbundle: + if wasbundle and mdfile: mddoc = _unzipattachments(mdfile) else: mddoc = mdfile @@ -260,8 +261,7 @@ def savetostorage(wopisrc, acctok, isclose, wopilock, onlyfetch=False): wasbundle = os.path.splitext(wopilock['fn'])[1] == '.zmd' bundlefile = attresponse = None if not disablezip or wasbundle: # in disablezip mode, preserve existing .zmd files but don't create new ones - bundlefile, attresponse = _getattachments(mddoc.decode(), wopilock['fn'].replace('.zmd', '.md'), - (wasbundle and not isclose)) + bundlefile, attresponse = _getattachments(mddoc.decode(), wopilock['fn'].replace('.zmd', '.md'), wasbundle) # WOPI PutFile for the file or the bundle if it already existed if (wasbundle ^ (not bundlefile)) or not isclose: diff --git a/src/bridge/etherpad.py b/src/bridge/etherpad.py index 49403a2b..531e8973 100644 --- a/src/bridge/etherpad.py +++ b/src/bridge/etherpad.py @@ -65,7 +65,7 @@ def _apicall(method, params, data=None, acctok=None, raiseonnonzerocode=True): return res -def getredirecturl(viewmode, wopisrc, acctok, docid, displayname): +def getredirecturl(viewmode, wopisrc, acctok, docid, _filename, displayname): '''Return a valid URL to the app for the given WOPI context''' # pass to Etherpad the required metadata for the save webhook try: From 25f05e5e32a80e29412cd3802e9970738a935774 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 26 Sep 2022 10:03:17 +0200 Subject: [PATCH 018/325] Bridged apps: fixed potential data loss We have observed sporadic failures when retrieving files from CodiMD, therefore we need to keep the file locked until we confirm the file is retrieved and stored safely. --- src/bridge/__init__.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 46c91e1d..a7fa6d37 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -348,12 +348,27 @@ def savedirty(self, openfile, wopisrc): openfile['toclose'] = {'invalid-lock': True} return None - WB.log.info('msg="SaveThread: saving file" token="%s" docid="%s"' % - (openfile['acctok'][-20:], openfile['docid'])) - WB.saveresponses[wopisrc] = WB.plugins[appname].savetostorage( - wopisrc, openfile['acctok'], _intersection(openfile['toclose']), wopilock) + # now save and log + WB.saveresponses[wopisrc] = WB.plugins[appname].savetostorage(wopisrc, openfile['acctok'], + _intersection(openfile['toclose']), wopilock) openfile['lastsave'] = int(time.time()) - openfile['tosave'] = False + if WB.saveresponses[wopisrc][1] >= http.client.BAD_REQUEST: + # this is hopefully transient, yet we need to try until we get the file back to storage: + # the updated lastsave time ensures next retry will happen after the saveinterval time + if 'still-dirty' not in openfile['toclose']: + # add a special key that will prevent close/unlock and refresh lock. If the refresh fails, + # the whole process will be retried at next round + openfile['toclose']['still-dirty'] = False + wopilock = wopic.refreshlock(wopisrc, openfile['acctok'], wopilock, toclose=openfile['toclose']) + WB.log.warning('msg="SaveThread: failed to save, will retry" token="%s" docid="%s" lasterror="%s" tocl="%s"' % + (openfile['acctok'][-20:], openfile['docid'], WB.saveresponses[wopisrc], wopilock['tocl'])) + else: + openfile['tosave'] = False + if 'still-dirty' in openfile['toclose']: # remove the special key above if present + openfile['toclose'].pop('still-dirty') + wopilock = wopic.refreshlock(wopisrc, openfile['acctok'], wopilock, toclose=openfile['toclose']) + WB.log.info('msg="SaveThread: file saved successfully" token="%s" docid="%s" tocl="%s"' % + (openfile['acctok'][-20:], openfile['docid'], wopilock['tocl'])) return wopilock def closewhenidle(self, openfile, wopisrc, wopilock): @@ -410,7 +425,7 @@ def cleanup(self, openfile, wopisrc, wopilock): try: wopic.refreshlock(wopisrc, openfile['acctok'], wopilock, toclose=openfile['toclose']) except wopic.InvalidLock: - WB.log.warning('msg="SaveThread: failed to refresh lock, will try again later" url="%s"' % wopisrc) + WB.log.warning('msg="SaveThread: failed to refresh lock, will retry" url="%s"' % wopisrc) @atexit.register From 62c59a7cadac2ff18ce2620c911f1b0b6c8eda7d Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 30 Sep 2022 17:00:54 +0200 Subject: [PATCH 019/325] Fixed URL decoding for display name --- src/wopiserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wopiserver.py b/src/wopiserver.py index 426a139d..f76431c2 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -300,7 +300,7 @@ def iopOpenInApp(): Wopi.log.warning('msg="iopOpenInApp: invalid viewmode parameter" client="%s" viewmode="%s" error="%s"' % (req.remote_addr, req.args.get('viewmode'), e)) return 'Missing or invalid viewmode argument', http.client.BAD_REQUEST - username = req.args.get('username', '') + username = url_unquote_plus(req.args.get('username', '')) # this needs to be a unique identifier: if missing (case of anonymous users), just generate a random string wopiuser = req.args.get('userid', utils.randomString(10)) folderurl = url_unquote_plus(req.args.get('folderurl', '%2F')) # defaults to `/` @@ -571,7 +571,7 @@ def cboxOpen_deprecated(): # backwards compatibility viewmode = utils.ViewMode.READ_WRITE if 'canedit' in req.args and req.args['canedit'].lower() == 'true' \ else utils.ViewMode.READ_ONLY - username = req.args.get('username', '') + username = url_unquote_plus(req.args.get('username', '')) folderurl = url_unquote_plus(req.args.get('folderurl', '%2F')) # defaults to `/` endpoint = req.args.get('endpoint', 'default') toproxy = req.args.get('proxy', 'false') == 'true' and filename[-1] == 'x' # if requested, only proxy OOXML files From ec9c68c4ca155397c1fb161330436b99b120fdec Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 30 Sep 2022 17:06:22 +0200 Subject: [PATCH 020/325] Fixed displayname encoding for bridged apps --- src/bridge/__init__.py | 2 +- src/bridge/etherpad.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index a7fa6d37..430d3c0f 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -201,7 +201,7 @@ def appopen(wopisrc, acctok, appname, viewmode): wopilock = app.loadfromstorage(filemd, wopisrc, acctok, None) redirurl = app.getredirecturl(viewmode, wopisrc, acctok, wopilock['doc'][1:], filemd['BaseFileName'], - urlparse.quote_plus(filemd['UserFriendlyName'])) + filemd['UserFriendlyName']) except app.AppFailure as e: # this can be raised by loadfromstorage or getredirecturl usermsg = str(e) if str(e) else 'Unable to load the app, please try again later or contact support' diff --git a/src/bridge/etherpad.py b/src/bridge/etherpad.py index 531e8973..c67768d2 100644 --- a/src/bridge/etherpad.py +++ b/src/bridge/etherpad.py @@ -85,9 +85,9 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, _filename, displayname): if viewmode in (utils.ViewMode.READ_ONLY, utils.ViewMode.VIEW_ONLY): # for read-only mode generate a read-only link res = _apicall('getReadOnlyID', {'padID': docid}, acctok=acctok) - return appexturl + '/p/%s?userName=%s' % (res['data']['readOnlyID'], displayname) + return appexturl + '/p/%s?userName=%s' % (res['data']['readOnlyID'], urlparse.quote_plus(displayname)) # return the URL to the pad (TODO if viewmode is PREVIEW) - return appexturl + '/p/%s?userName=%s' % (docid, displayname) + return appexturl + '/p/%s?userName=%s' % (docid, urlparse.quote_plus(displayname)) # Cloud storage to Etherpad From 9e1844bda17525f03be16e0dfe0c6cfd2651ce82 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 29 Sep 2022 10:25:40 +0200 Subject: [PATCH 021/325] Revert #85 and spell out how UnlockAndRelock is implemented Also use UnlockAndRelock in bridged apps to be fully WOPI specs compliant --- src/bridge/wopiclient.py | 10 +++++----- src/core/wopi.py | 5 +---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/bridge/wopiclient.py b/src/bridge/wopiclient.py index c0e09365..1f8e0e3f 100644 --- a/src/bridge/wopiclient.py +++ b/src/bridge/wopiclient.py @@ -93,8 +93,8 @@ def getlock(wopisrc, acctok): raise InvalidLock(e) -def _getheadersforrefreshlock(acctok, wopilock, digest, toclose): - '''Helper function for refreshlock to generate the old and new lock structures''' +def _getheadersforrelock(acctok, wopilock, digest, toclose): + '''Helper function for relock to generate the old and new lock structures''' newlock = json.loads(json.dumps(wopilock)) # this is a hack for a deep copy if toclose: # we got the full 'toclose' dict, push it as is @@ -105,7 +105,7 @@ def _getheadersforrefreshlock(acctok, wopilock, digest, toclose): if digest and wopilock['dig'] != digest: newlock['dig'] = digest return { - 'X-Wopi-Override': 'REFRESH_LOCK', + 'X-Wopi-Override': 'LOCK', 'X-WOPI-OldLock': json.dumps(wopilock), 'X-WOPI-Lock': json.dumps(newlock) }, newlock @@ -113,7 +113,7 @@ def _getheadersforrefreshlock(acctok, wopilock, digest, toclose): def refreshlock(wopisrc, acctok, wopilock, digest=None, toclose=None): '''Refresh an existing WOPI lock. Returns the new lock if successful, None otherwise''' - h, newlock = _getheadersforrefreshlock(acctok, wopilock, digest, toclose) + h, newlock = _getheadersforrelock(acctok, wopilock, digest, toclose) res = request(wopisrc, acctok, 'POST', headers=h) if res.status_code == http.client.OK: return newlock @@ -133,7 +133,7 @@ def refreshlock(wopisrc, acctok, wopilock, digest=None, toclose=None): if digest: wopilock['dig'] = currlock['dig'] # retry with the newly got lock - h, newlock = _getheadersforrefreshlock(acctok, wopilock, digest, toclose) + h, newlock = _getheadersforrelock(acctok, wopilock, digest, toclose) res = request(wopisrc, acctok, 'POST', headers=h) if res.status_code == http.client.OK: return newlock diff --git a/src/core/wopi.py b/src/core/wopi.py index b7c7d493..cb57ce95 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -218,10 +218,6 @@ def setLock(fileid, reqheaders, acctok): (op.title(), fn, flask.request.args['access_token'][-20:], e)) try: - # If LOCK and oldLock provided this is an UnLockAndRelock operation - if op == 'LOCK' and oldLock: - st.unlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(oldLock)) - # LOCK or REFRESH_LOCK: atomically set the lock to the given one, including the expiration time, # and return conflict response if the file was already locked st.setlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(lock)) @@ -251,6 +247,7 @@ def setLock(fileid, reqheaders, acctok): # get the lock that was set if not retrievedLock: retrievedLock, lockHolder = utils.retrieveWopiLock(fileid, op, lock, acctok) + # validate against either the given lock (RefreshLock case) or the given old lock (UnlockAndRelock case) if retrievedLock and not utils.compareWopiLocks(retrievedLock, (oldLock if oldLock else lock)): # lock mismatch, the WOPI client is supposed to acknowledge the existing lock # or deny write access to the file From f1afc6af4e61654ce16b0d312ac286cc637a82d1 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 29 Sep 2022 10:27:54 +0200 Subject: [PATCH 022/325] Enforce checking the old lock value on refreshLock, cf. cs3org/cs3apis#183 Correspondingly updated the test suite and the minimum CS3APIs version --- requirements.txt | 2 +- src/core/commoniface.py | 21 ++++++++++++--------- src/core/cs3iface.py | 8 ++++++-- src/core/localiface.py | 6 +++--- src/core/wopi.py | 4 ++-- src/core/xrootiface.py | 6 +++--- test/test_storageiface.py | 5 ++++- 7 files changed, 31 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index 471d18c7..7711cfda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ PyJWT requests more_itertools prometheus-flask-exporter -cs3apis>=0.1.dev95 +cs3apis>=0.1.dev101 waitress diff --git a/src/core/commoniface.py b/src/core/commoniface.py index 86fc8158..fcbee085 100644 --- a/src/core/commoniface.py +++ b/src/core/commoniface.py @@ -81,15 +81,18 @@ def encodeinode(endpoint, inode): return endpoint + '-' + urlsafe_b64encode(inode.encode()).decode() -def validatelock(filepath, appname, oldlock, op, log): +def validatelock(filepath, appname, oldlock, oldvalue, op, log): '''Common logic for validating locks in the xrootd and local storage interfaces. Duplicates some logic implemented in Reva for the cs3 storage interface''' - if not oldlock: - log.warning('msg="Failed to %s" filepath="%s" appname="%s" reason="%s"' % - (op, filepath, appname, 'File was not locked or lock had expired')) - raise IOError('File was not locked or lock had expired') - if oldlock['app_name'] != 'wopi' and appname != 'wopi' and oldlock['app_name'] and appname \ - and oldlock['app_name'] != appname: + try: + if not oldlock: + raise IOError('File was not locked or lock had expired') + if oldvalue and oldlock['lock_id'] != oldvalue: + raise IOError('Existing lock payload does not match') + if appname and oldlock['app_name'] != appname \ + and oldlock['app_name'] != 'wopi' and appname != 'wopi': # TODO deprecated, to be removed after CERNBox rollout + raise IOError('File is locked by %s' % oldlock['app_name']) + except IOError as e: log.warning('msg="Failed to %s" filepath="%s" appname="%s" reason="%s"' % - (op, filepath, appname, 'File is locked by %s' % oldlock['app_name'])) - raise IOError('File is locked by %s' % oldlock['app_name']) + (op, filepath, appname, e)) + raise diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index ca3064ef..89d9a194 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -222,13 +222,17 @@ def getlock(endpoint, filepath, userid): } -def refreshlock(endpoint, filepath, userid, appname, value): +def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): '''Refresh the lock metadata for the given filepath''' reference = _getcs3reference(endpoint, filepath) lock = cs3spr.Lock(type=cs3spr.LOCK_TYPE_WRITE, app_name=appname, lock_id=value, expiration={'seconds': int(time.time() + ctx['lockexpiration'])}) - req = cs3sp.RefreshLockRequest(ref=reference, lock=lock) + req = cs3sp.RefreshLockRequest(ref=reference, lock=lock, existing_lock_id=oldvalue) res = ctx['cs3gw'].RefreshLock(request=req, metadata=[('x-access-token', userid)]) + if res.status.code in [cs3code.CODE_FAILED_PRECONDITION, cs3code.CODE_ABORTED]: + log.info('msg="Failed precondition on refreshlock" filepath="%s" appname="%s" trace="%s" reason="%s"' % + (filepath, appname, res.status.trace, res.status.message.replace('"', "'"))) + raise IOError(common.EXCL_ERROR) if res.status.code != cs3code.CODE_OK: log.warning('msg="Failed to refreshlock" filepath="%s" appname="%s" value="%s" trace="%s" code="%s" reason="%s"' % (filepath, appname, value, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) diff --git a/src/core/localiface.py b/src/core/localiface.py index 724e9966..ce2c5243 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -175,9 +175,9 @@ def getlock(endpoint, filepath, _userid): return None -def refreshlock(endpoint, filepath, userid, appname, value): +def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): '''Refresh the lock value as an xattr on behalf of the given userid''' - common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), 'refreshlock', log) + common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), oldvalue, 'refreshlock', log) # this is non-atomic, but if we get here the lock was already held log.debug('msg="Invoked refreshlock" filepath="%s" value="%s"' % (filepath, value)) setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value), LOCK) @@ -185,7 +185,7 @@ def refreshlock(endpoint, filepath, userid, appname, value): def unlock(endpoint, filepath, userid, appname, value): '''Remove the lock as an xattr on behalf of the given userid''' - common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), 'unlock', log) + common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), None, 'unlock', log) log.debug('msg="Invoked unlock" filepath="%s" value="%s"' % (filepath, value)) rmxattr(endpoint, filepath, '0:0', common.LOCKKEY, LOCK) diff --git a/src/core/wopi.py b/src/core/wopi.py index cb57ce95..b2d845c8 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -255,10 +255,10 @@ def setLock(fileid, reqheaders, acctok): 'The file is locked by %s' % (lockHolder if lockHolder != 'wopi' else 'another online editor')) - # else it's our own lock, refresh it and return + # else it's our own lock, refresh it (rechecking the oldLock if necessary, for atomicity) and return try: st.refreshlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], - utils.encodeLock(lock)) + utils.encodeLock(lock), utils.encodeLock(oldLock) if oldLock else None) return utils.makeLockSuccessResponse(op, fn, lock, 'v%s' % statInfo['etag']) except IOError as rle: # this is unexpected now diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 1d2c78cb..4095c166 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -308,9 +308,9 @@ def getlock(endpoint, filepath, userid): return None # no pre-existing lock found, or error attempting to read it: assume it does not exist -def refreshlock(endpoint, filepath, userid, appname, value): +def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): '''Refresh the lock value as an xattr''' - common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), 'refreshlock', log) + common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), oldvalue, 'refreshlock', log) log.debug('msg="Invoked refreshlock" filepath="%s" value="%s"' % (filepath, value)) # this is non-atomic, but the lock was already held setxattr(endpoint, filepath, userid, common.LOCKKEY, common.genrevalock(appname, value), None) @@ -318,7 +318,7 @@ def refreshlock(endpoint, filepath, userid, appname, value): def unlock(endpoint, filepath, userid, appname, value): '''Remove a lock as an xattr''' - common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), 'unlock', log) + common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), None, 'unlock', log) log.debug('msg="Invoked unlock" filepath="%s" value="%s"' % (filepath, value)) rmxattr(endpoint, filepath, userid, common.LOCKKEY, None) diff --git a/test/test_storageiface.py b/test/test_storageiface.py index b725d234..d3617b31 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -241,7 +241,10 @@ def test_refresh_lock(self): self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'testlock') self.assertIn('File was not locked', str(context.exception)) self.storage.setlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'testlock') - self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'testlock2') + with self.assertRaises(IOError) as context: + self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'newlock', 'mismatched') + self.assertIn('Existing lock payload does not match', str(context.exception)) + self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'testlock2', 'testlock') l = self.storage.getlock(self.endpoint, self.homepath + '/testrlock', self.userid) # noqa: E741 self.assertIsInstance(l, dict) self.assertEqual(l['lock_id'], 'testlock2') From 1e13a24a390ee08faf66bb47398933033bcc228b Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 3 Oct 2022 10:54:27 +0200 Subject: [PATCH 023/325] Minor refactoring --- src/core/commoniface.py | 8 ++++++-- src/core/localiface.py | 2 +- test/test_storageiface.py | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/core/commoniface.py b/src/core/commoniface.py index fcbee085..5211950b 100644 --- a/src/core/commoniface.py +++ b/src/core/commoniface.py @@ -19,6 +19,9 @@ # standard error thrown when attempting to overwrite a file/xattr in O_EXCL mode EXCL_ERROR = 'File exists and islock flag requested' +# error thrown when relocking a file and the payload does not match +LOCK_MISMATCH_ERROR = 'Existing lock payload does not match' + # standard error thrown when attempting an operation without the required access rights ACCESS_ERROR = 'Operation not permitted' @@ -88,11 +91,12 @@ def validatelock(filepath, appname, oldlock, oldvalue, op, log): if not oldlock: raise IOError('File was not locked or lock had expired') if oldvalue and oldlock['lock_id'] != oldvalue: - raise IOError('Existing lock payload does not match') + raise IOError(LOCK_MISMATCH_ERROR) if appname and oldlock['app_name'] != appname \ - and oldlock['app_name'] != 'wopi' and appname != 'wopi': # TODO deprecated, to be removed after CERNBox rollout + and oldlock['app_name'] != 'wopi' and appname != 'wopi': # TODO deprecated, to be removed after CERNBox rollout raise IOError('File is locked by %s' % oldlock['app_name']) except IOError as e: log.warning('msg="Failed to %s" filepath="%s" appname="%s" reason="%s"' % (op, filepath, appname, e)) raise + diff --git a/src/core/localiface.py b/src/core/localiface.py index ce2c5243..771a8c69 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -131,7 +131,7 @@ def getxattr(_endpoint, filepath, _userid, key): try: return os.getxattr(_getfilepath(filepath), 'user.' + key).decode('UTF-8') except OSError as e: - log.warn('msg="Failed to getxattr or missing key" filepath="%s" key="%s" exception="%s"' % (filepath, key, e)) + log.warning('msg="Failed to getxattr or missing key" filepath="%s" key="%s" exception="%s"' % (filepath, key, e)) return None diff --git a/test/test_storageiface.py b/test/test_storageiface.py index d3617b31..7bb62b34 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -17,7 +17,7 @@ from threading import Thread sys.path.append('src') # for tests out of the git repo sys.path.append('/app') # for tests within the Docker image -from core.commoniface import EXCL_ERROR, ENOENT_MSG # noqa: E402 +from core.commoniface import EXCL_ERROR, LOCK_MISMATCH_ERROR, ENOENT_MSG # noqa: E402 databuf = b'ebe5tresbsrdthbrdhvdtr' @@ -243,7 +243,7 @@ def test_refresh_lock(self): self.storage.setlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'testlock') with self.assertRaises(IOError) as context: self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'newlock', 'mismatched') - self.assertIn('Existing lock payload does not match', str(context.exception)) + self.assertIn(LOCK_MISMATCH_ERROR, str(context.exception)) self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'testlock2', 'testlock') l = self.storage.getlock(self.endpoint, self.homepath + '/testrlock', self.userid) # noqa: E741 self.assertIsInstance(l, dict) From 0be97ddca5cade957fc76f230498d62ec29eecee Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 4 Oct 2022 14:04:53 +0200 Subject: [PATCH 024/325] CodiMD: use an always-false parameter for controlling the embedding, as opposed to always-true --- src/bridge/codimd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 79babf4e..ada8680b 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -65,7 +65,7 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname): 'accessToken': acctok, 'apiKey': apikey, 'displayName': displayname, - 'allowEmbedding': os.path.splitext(filename)[1] == '.zmd', + 'disableEmbedding': os.path.splitext(filename)[1] != '.zmd', } return f'{appexturl}/{docid}?{mode}&{urlparse.urlencode(params)}' From 7b3b6990f085365ae4d456d55cb523a8bd36a084 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 4 Oct 2022 14:18:43 +0200 Subject: [PATCH 025/325] Preparing release --- CHANGELOG.md | 11 ++++++++++- README.md | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7c11db0..e1ecd0fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ ## Changelog for the WOPI server +### Tue Oct 4 2022 - v9.1.0 +- Introduced support for PREVIEW mode (#82) +- Improved UnlockAndRelock logic (#85, #87) +- Switched to python-alpine docker image (#88) +- Introduced further branding options in CheckFileInfo +- Further improvements in the bridged apps logic +- Added more logging and a new endpoint to monitor + conflicted sessions + ### Thu Sep 1 2022 - v9.0.0 - Refactored and strengthened save workflow for bridged applications, and simplified lock metadata (#80) @@ -8,7 +17,7 @@ - Refactored PutFile logic when handling conflict files (#78) - Improved support for Spaces in Reva (#79) - Implemented save workflow for Etherpad documents (#81) - Fixed direct download in case of errors +- Fixed direct download in case of errors - Updated dependencies and documentation ### Thu Jun 16 2022 - v8.3.0 diff --git a/README.md b/README.md index 323c6d93..ffee32ab 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Contributors: - Gianmaria Del Monte (@gmgigi96) - Klaas Freitag (@dragotin) - Jörn Friedrich Dreyer (@butonic) +- Michael Barz (@micbar) Initial revision: December 2016
First production version for CERNBox: September 2017 (presented at [oCCon17](https://occon17.owncloud.org) - [slides](https://www.slideshare.net/giuseppelopresti/collaborative-editing-and-more-in-cernbox))
From aa59bb6c2b60410bdac22738ba8f68967169847b Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 5 Oct 2022 08:37:49 +0200 Subject: [PATCH 026/325] Consider a session as "resolved" also when it successfully saves a file --- src/core/wopiutils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 0c241de8..78331142 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -358,7 +358,7 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, endpoint else: srv.conflictsessions['pending'][session] = 1 log.warning('msg="Returning conflict" lockop="%s" user="%s" filename="%s" token="%s" sessionId="%s" lock="%s" ' - 'oldlock="%s" retrievedlock="%s" fileage="%s" reason="%s"' % + 'oldlock="%s" retrievedlock="%s" fileage="%1.1f" reason="%s"' % (operation.title(), user, filename, flask.request.args['access_token'][-20:], session, lock, oldlock, retrievedlock, time.time() - savetime, (reason['message'] if reason else 'NA'))) @@ -383,6 +383,11 @@ def makeLockSuccessResponse(operation, filename, lock, version): def storeWopiFile(acctok, retrievedlock, xakey, targetname=''): '''Saves a file from an HTTP request to the given target filename (defaulting to the access token's one), and stores the save time as an xattr. Throws IOError in case of any failure''' + session = flask.request.headers.get('X-WOPI-SessionId') + if session in srv.conflictsessions['pending']: + counter = srv.conflictsessions['pending'].pop(session) + srv.conflictsessions['resolved'][session] = counter + if not targetname: targetname = acctok['filename'] st.writefile(acctok['endpoint'], targetname, acctok['userid'], flask.request.get_data(), encodeLock(retrievedlock)) From c1620b48e088e1c518de26968c0a520e7c77f880 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 5 Oct 2022 08:38:22 +0200 Subject: [PATCH 027/325] Updated for the release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1ecd0fa..6fbb5a09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Changelog for the WOPI server -### Tue Oct 4 2022 - v9.1.0 +### Wed Oct 5 2022 - v9.1.0 - Introduced support for PREVIEW mode (#82) - Improved UnlockAndRelock logic (#85, #87) - Switched to python-alpine docker image (#88) From bf7ddc771a22dc1d56831a315428dff28c3d172b Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 4 Oct 2022 14:18:11 +0200 Subject: [PATCH 028/325] Bridged apps: added debug line --- src/bridge/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 430d3c0f..3b3e3565 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -118,6 +118,7 @@ def _validateappname(appname): for p in WB.plugins.values(): if appname.lower() in p.appname.lower(): return p.appname + WB.log.debug('msg="BridgeSave: unknown application" appname="%s" plugins="%s"' % (appname, WB.plugins.values())) raise ValueError From ef776928cbc2cd2c8167eb6ab202dc01677aba2e Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 5 Oct 2022 15:27:44 +0200 Subject: [PATCH 029/325] Bridged apps: truncate content in logs --- src/bridge/codimd.py | 2 +- src/bridge/etherpad.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index ada8680b..33abb4b5 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -130,7 +130,7 @@ def _fetchfromcodimd(wopilock, acctok): res = requests.get(appurl + ('/' if wopilock['doc'][0] != '/' else '') + wopilock['doc'] + '/download', verify=sslverify) if res.status_code != http.client.OK: log.error('msg="Unable to fetch document from CodiMD" token="%s" response="%d: %s"' % - (acctok[-20:], res.status_code, res.content.decode())) + (acctok[-20:], res.status_code, res.content.decode()[:50])) raise AppFailure return res.content except requests.exceptions.ConnectionError as e: diff --git a/src/bridge/etherpad.py b/src/bridge/etherpad.py index c67768d2..dc68c28d 100644 --- a/src/bridge/etherpad.py +++ b/src/bridge/etherpad.py @@ -117,8 +117,8 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): params={'apikey': apikey}, verify=sslverify) if res.status_code != http.client.OK: - log.error('msg="Unable to push document to Etherpad" token="%s" padid="%s" response="%d: %s" content="%s"' % - (acctok[-20:], docid, res.status_code, res.content.decode(), epfile.decode())) + log.error('msg="Unable to push document to Etherpad" token="%s" padid="%s" response="%d: %s"' % + (acctok[-20:], docid, res.status_code, res.content.decode())) raise AppFailure log.info('msg="Pushed document to Etherpad" padid="%s" token="%s"' % (docid, acctok[-20:])) except requests.exceptions.ConnectionError as e: @@ -139,7 +139,7 @@ def _fetchfrometherpad(wopilock, acctok): verify=sslverify) if res.status_code != http.client.OK: log.error('msg="Unable to fetch document from Etherpad" token="%s" response="%d: %s"' % - (acctok[-20:], res.status_code, res.content.decode())) + (acctok[-20:], res.status_code, res.content.decode()[:50])) raise AppFailure return res.content except requests.exceptions.ConnectionError as e: From ddf5635c4a5f3c28e5d342f021f845e39f4dac48 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 5 Oct 2022 22:59:21 +0200 Subject: [PATCH 030/325] CodiMD: lower case for query params + dropped remaining references to apikey --- src/bridge/codimd.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 33abb4b5..de32ff33 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -26,7 +26,6 @@ # initialized by the main class or by the init method appurl = None appexturl = None -apikey = None log = None sslverify = None disablezip = None @@ -40,10 +39,8 @@ def init(_appurl, _appinturl, _apikey): '''Initialize global vars from the environment''' global appurl global appexturl - global apikey appexturl = _appurl appurl = _appinturl - apikey = _apikey try: # CodiMD integrates Prometheus metrics, let's probe if they exist res = requests.head(appurl + '/metrics/codimd', verify=sslverify) @@ -63,9 +60,8 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname): params = { 'wopiSrc': wopisrc, 'accessToken': acctok, - 'apiKey': apikey, + 'disableEmbedding': ('%s' % (os.path.splitext(filename)[1] != '.zmd')).lower(), 'displayName': displayname, - 'disableEmbedding': os.path.splitext(filename)[1] != '.zmd', } return f'{appexturl}/{docid}?{mode}&{urlparse.urlencode(params)}' From c7990c33493cf67854c9656d7823d180dfea25e3 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 5 Oct 2022 23:02:59 +0200 Subject: [PATCH 031/325] Bridged apps: added check to prevent misconfigurations --- src/bridge/__init__.py | 10 +++++++--- src/wopiserver.py | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 3b3e3565..29e1aa0a 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -78,7 +78,11 @@ def loadplugin(cls, appname, appurl, appinturl, apikey): '''Load plugin for the given appname, if supported by the bridge service''' p = appname.lower() if p in cls.plugins: - # already initialized + # already initialized, check that the app URL matches: the current model does not support multiple app backends + if appurl != cls.plugins[p].appexturl: + cls.log.warning('msg="Attempt to use plugin with another appurl" client="%s" app="%s" appurl="%s"' % + (flask.request.remote_addr, appname, appurl)) + raise KeyError(appname) return if not issupported(appname): raise ValueError(appname) @@ -91,8 +95,8 @@ def loadplugin(cls, appname, appurl, appinturl, apikey): cls.plugins[p].init(appurl, appinturl, apikey) cls.log.info('msg="Imported plugin for application" app="%s" plugin="%s"' % (p, cls.plugins[p])) except Exception as e: - cls.log.info('msg="Failed to initialize plugin" app="%s" URL="%s" exception="%s"' % - (p, appinturl, e)) + cls.log.warning('msg="Failed to initialize plugin" app="%s" URL="%s" exception="%s"' % + (p, appinturl, e)) cls.plugins.pop(p, None) # regardless which step failed, this will remove the failed plugin raise ValueError(appname) diff --git a/src/wopiserver.py b/src/wopiserver.py index f76431c2..699dae20 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -320,6 +320,8 @@ def iopOpenInApp(): bridge.WB.loadplugin(appname, appurl, appinturl, apikey) except ValueError: return 'Failed to load WOPI bridge plugin for %s' % appname, http.client.INTERNAL_SERVER_ERROR + except KeyError: + return 'Bridged app %s already configured with a different appurl' % appname, http.client.NOT_IMPLEMENTED try: userid = storage.getuseridfromcreds(usertoken, wopiuser) From 7e972ca8ae38aa2b0230c83c27d982d0173196a9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 6 Oct 2022 09:18:54 +0200 Subject: [PATCH 032/325] Linting --- src/bridge/__init__.py | 2 +- src/bridge/codimd.py | 5 ++--- src/core/commoniface.py | 1 - src/core/xrootiface.py | 1 + 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 29e1aa0a..fe0c4b7b 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -233,7 +233,7 @@ def appsave(docid): WB.log.error('msg="BridgeSave: missing metadata" address="%s" headers="%s" args="%s" error="%s"' % (flask.request.remote_addr, flask.request.headers, flask.request.args, e)) return wopic.jsonify('Missing metadata, could not save. %s' % RECOVER_MSG), http.client.BAD_REQUEST - except ValueError as e: + except ValueError: WB.log.error('msg="BridgeSave: unknown application" address="%s" appheader="%s" args="%s"' % (flask.request.remote_addr, flask.request.headers.get(BRIDGED_APP_HEADER), flask.request.args)) # temporary override diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index de32ff33..3b710a8e 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -66,8 +66,7 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname): return f'{appexturl}/{docid}?{mode}&{urlparse.urlencode(params)}' # read-only mode: first check if we have a CodiMD redirection - res = requests.head(appurl + '/' + docid, - verify=sslverify) + res = requests.head(appurl + '/' + docid, verify=sslverify) if res.status_code == http.client.FOUND: return '%s/s/%s' % (appexturl, urlparse.urlsplit(res.next.url).path.split('/')[-1]) # we used to redirect to publish mode or normal view to quickly jump in slide mode depending on the content, @@ -115,7 +114,7 @@ def _unzipattachments(inputbuf): return mddoc -#def _isslides(doc): +# def _isslides(doc): # '''Heuristically look for signatures of slides in the header of a md document''' # return doc[:9].decode() == '---\ntitle' or doc[:8].decode() == '---\ntype' or doc[:16].decode() == '---\nslideOptions' diff --git a/src/core/commoniface.py b/src/core/commoniface.py index 5211950b..0b00995f 100644 --- a/src/core/commoniface.py +++ b/src/core/commoniface.py @@ -99,4 +99,3 @@ def validatelock(filepath, appname, oldlock, oldvalue, op, log): log.warning('msg="Failed to %s" filepath="%s" appname="%s" reason="%s"' % (op, filepath, appname, e)) raise - diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 4095c166..d0c54cd6 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -27,6 +27,7 @@ homepath = None timeout = None + def init(inconfig, inlog): '''Init module-level variables''' global config # pylint: disable=global-statement From 7ec5b4a5e1b93cb8a5211c2cd3e221c200fab4b2 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Thu, 6 Oct 2022 10:49:54 +0200 Subject: [PATCH 033/325] feat: add option to use file or stream handler for logging --- docker/etc/wopiserver.conf | 1 + src/wopiserver.py | 22 ++++++++++++++++------ wopiserver.conf | 9 +++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/docker/etc/wopiserver.conf b/docker/etc/wopiserver.conf index a4f3b91e..6711120d 100644 --- a/docker/etc/wopiserver.conf +++ b/docker/etc/wopiserver.conf @@ -13,6 +13,7 @@ wopilockexpiration = 1800 # Logging level. Debug enables the Flask debug mode as well. # Valid values are: Debug, Info, Warning, Error. loglevel = Debug +loghandler = file [security] usehttps = no diff --git a/src/wopiserver.py b/src/wopiserver.py index 699dae20..f5125a4c 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -86,18 +86,28 @@ def init(cls): hostname = os.environ.get('HOST_HOSTNAME') if not hostname: hostname = socket.gethostname() + # read the configuration + cls.config = configparser.ConfigParser() + with open('/etc/wopi/wopiserver.defaults.conf') as fdef: + cls.config.read_file(fdef) + cls.config.read('/etc/wopi/wopiserver.conf') # configure the logging - loghandler = logging.FileHandler('/var/log/wopi/wopiserver.log') + lhandler = cls.config.get('general', 'loghandler', fallback='file').lower() + if lhandler == 'stream': + logdest = cls.config.get('general', 'logdest', fallback='stdout').lower() + if logdest == "stdout": + logdest = sys.stdout + else: + logdest = sys.stderr + loghandler = logging.StreamHandler(logdest) + else: + logdest = cls.config.get('general', 'logdest', fallback='/var/log/wopi/wopiserver.log') + loghandler = logging.FileHandler(logdest) loghandler.setFormatter(logging.Formatter( fmt='{"time": "%(asctime)s.%(msecs)03d", "host": "' + hostname + '", "level": "%(levelname)s", "process": "%(name)s", %(message)s}', datefmt='%Y-%m-%dT%H:%M:%S')) cls.app.logger.handlers = [loghandler] - # read the configuration - cls.config = configparser.ConfigParser() - with open('/etc/wopi/wopiserver.defaults.conf') as fdef: - cls.config.read_file(fdef) - cls.config.read('/etc/wopi/wopiserver.conf') # load the requested storage layer storage_layer_import(cls.config.get('general', 'storagetype')) # prepare the Flask web app diff --git a/wopiserver.conf b/wopiserver.conf index d0d49d0c..26b51f82 100644 --- a/wopiserver.conf +++ b/wopiserver.conf @@ -17,6 +17,15 @@ port = 8880 # Valid values are: Debug, Info, Warning, Error. loglevel = Info +# Logging handler. Sets the log handler to use. +# Valid values are: file, stream. +loghandler = file + +# Logging destination. +# Valid values if 'loghandler = file' are: any existing file path. +# Valid values if 'loghandler = stream' are: stdout, stderr. +#logdest = /var/log/wopi/wopiserver.log + # URL of your WOPI server or your HA proxy in front of it #wopiurl = https://your-wopi-server.org:8443 From 0ec21d7c0d6e40cc7529c8eec67faf0867ed59b2 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 6 Oct 2022 18:09:45 +0200 Subject: [PATCH 034/325] Log remote address from traefik when present --- src/core/wopiutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 78331142..6a8e01ea 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -126,7 +126,7 @@ def validateAndLogHeaders(op): log.debug('msg="%s: client context" user="%s" filename="%s" token="%s" client="%s" deviceId="%s" reqId="%s" sessionId="%s" ' 'app="%s" appEndpoint="%s" correlationId="%s" wopits="%s"' % (op.title(), acctok['userid'][-20:], acctok['filename'], - flask.request.args['access_token'][-20:], flask.request.remote_addr, + flask.request.args['access_token'][-20:], flask.request.headers.get('X-Real-Ip', flask.request.remote_addr), flask.request.headers.get('X-WOPI-DeviceId'), flask.request.headers.get('X-Request-Id'), flask.request.headers.get('X-WOPI-SessionId'), flask.request.headers.get('X-WOPI-RequestingApplication'), flask.request.headers.get('X-WOPI-AppEndpoint'), flask.request.headers.get('X-WOPI-CorrelationId'), wopits)) From 54a3891d0fc7f6db6f7532ef609fe2d2a703f9fa Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 6 Oct 2022 21:38:32 +0200 Subject: [PATCH 035/325] Moved monitoring scripts to a separate repo --- Makefile | 2 +- cernbox-wopi-server.spec | 1 - mon/wopi_grafana_feeder.py2 | 162 ----------------------------------- mon/wopi_max_concurrency.py2 | 112 ------------------------ 4 files changed, 1 insertion(+), 276 deletions(-) delete mode 100755 mon/wopi_grafana_feeder.py2 delete mode 100755 mon/wopi_max_concurrency.py2 diff --git a/Makefile b/Makefile index 590aa17a..8bf09cdd 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -FILES_TO_RPM = src mon tools wopiserver.conf wopiserver.service wopiserver.logrotate +FILES_TO_RPM = src tools wopiserver.conf wopiserver.service wopiserver.logrotate SPECFILE = $(shell find . -type f -name *.spec) VERSREL = $(shell git describe | sed 's/^v//') VERSION = $(shell echo ${VERSREL} | cut -d\- -f 1) diff --git a/cernbox-wopi-server.spec b/cernbox-wopi-server.spec index 07da8bff..03ceb5cd 100644 --- a/cernbox-wopi-server.spec +++ b/cernbox-wopi-server.spec @@ -55,7 +55,6 @@ install -m 644 src/cs3iface.py %buildroot/%_python_lib/cs3iface.py install -m 644 wopiserver.service %buildroot/usr/lib/systemd/system/wopiserver.service install -m 644 wopiserver.conf %buildroot/etc/wopi/wopiserver.defaults.conf install -m 644 wopiserver.logrotate %buildroot/etc/logrotate.d/cernbox-wopi-server -install -m 755 mon/wopi_grafana_feeder.py %buildroot/usr/bin/wopi_grafana_feeder.py install -m 755 tools/wopicheckfile.py %buildroot/usr/bin/wopicheckfile.py install -m 755 tools/wopilistopenfiles.sh %buildroot/usr/bin/wopilistopenfiles.sh install -m 755 tools/wopiopen.py %buildroot/usr/bin/wopiopen.py diff --git a/mon/wopi_grafana_feeder.py2 b/mon/wopi_grafana_feeder.py2 deleted file mode 100755 index 2fbd1f05..00000000 --- a/mon/wopi_grafana_feeder.py2 +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/python -''' -wopi_grafana_feeder.py - -A daemon pushing CERNBox WOPI monitoring data to Grafana. -TODO: make it a collectd plugin. References: -https://collectd.org/documentation/manpages/collectd-python.5.shtml -https://blog.dbrgn.ch/2017/3/10/write-a-collectd-python-plugin/ -https://github.com/dbrgn/collectd-python-plugins - -author: Giuseppe.LoPresti@cern.ch -CERN/IT-ST -''' - -import fileinput -import socket -import time -import pickle -import struct -import datetime -import getopt -import sys - -CARBON_TCPPORT = 2004 -carbonHost = '' -verbose = False -prefix = 'cernbox.wopi.' + socket.gethostname().split('.')[0] -epoch = datetime.datetime(1970, 1, 1) - - -def usage(exitCode): - '''prints usage''' - print 'Usage : cat | ' + sys.argv[0] + ' [-h|--help] -g|--grafanahost ' - sys.exit(exitCode) - -def send_metric(data): - '''send data to grafana using the pickle protocol''' - payload = pickle.dumps(data, protocol=2) - header = struct.pack("!L", len(payload)) - message = header + payload - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((carbonHost, CARBON_TCPPORT)) - sock.sendall(message) - sock.close() - -def get_wopi_metrics(data): - '''Parse WOPI usage metrics''' - for line in data: - if data.isfirstline(): - logdate = line.split('T')[0].split('-') # keeps the date until 'T', splits - timestamp = (datetime.datetime(int(logdate[0]), int(logdate[1]), int(logdate[2]), 1, 0, 0) - epoch).total_seconds() + time.altzone - errors = 0 - users = {} - openfiles = {} - openfiles['docx'] = {} - openfiles['xlsx'] = {} - openfiles['pptx'] = {} - openfiles['odt'] = {} - openfiles['ods'] = {} - openfiles['odp'] = {} - openfiles['md'] = {} - openfiles['zmd'] = {} - openfiles['txt'] = {} - wrfiles = {} - wrfiles['docx'] = {} - wrfiles['xlsx'] = {} - wrfiles['pptx'] = {} - wrfiles['odt'] = {} - wrfiles['ods'] = {} - wrfiles['odp'] = {} - wrfiles['md'] = {} - wrfiles['zmd'] = {} - wrfiles['txt'] = {} - collab = 0 - try: - if ' ERROR ' in line: - errors += 1 - # all opened files - elif 'CheckFileInfo' in line: - # count of unique users - l = line.split() - u = l[4].split('=')[1] - if u in users.keys(): - users[u] += 1 - else: - users[u] = 1 - # count of open files per type: look for the file extension - fname = line[line.find('filename=')+10:line.rfind('fileid=')-2] - fext = fname[fname.rfind('.')+1:] - if fext not in openfiles: - openfiles[fext] = {} - if fname in openfiles[fext]: - openfiles[fext][fname] += 1 - else: - openfiles[fext][fname] = 1 - # files opened for write - elif 'successfully written' in line: - # count of written files - fname = line[line.find('filename=')+10:line.rfind('token=')-2] - fext = fname[fname.rfind('.')+1:] - if fname in wrfiles[fext]: - wrfiles[fext][fname] += 1 - else: - wrfiles[fext][fname] = 1 - # collaborative editing sessions - elif 'Collaborative editing detected' in line: - collab += 1 - # we could extract the filename and the users list for further statistics - except Exception: - if verbose: - print 'Error occurred at line: %s' % line - raise - - if 'timestamp' not in locals(): - # the file was empty, nothing to do - return - # prepare data for grafana - output = [] - output.append(( prefix + '.errors', (int(timestamp), errors) )) - output.append(( prefix + '.users', (int(timestamp), len(users)) )) - # get the top user by sorting the users dict by values instead of by keys - if len(users) > 0: - top = sorted(users.iteritems(), key=lambda (k, v): (v, k))[-1][1] - output.append(( prefix + '.topuser', (int(timestamp), int(top)) )) - for fext in openfiles: - output.append(( prefix + '.openfiles.' + fext, (int(timestamp), len(openfiles[fext])) )) - for fext in wrfiles: - output.append(( prefix + '.writtenfiles.' + fext, (int(timestamp), len(wrfiles[fext])) )) - output.append(( prefix + '.collab', (int(timestamp), collab) )) - # send and print all collected data - send_metric(output) - if verbose: - print output - - -# first parse options -try: - options, args = getopt.getopt(sys.argv[1:], 'hvg:', ['help', 'verbose', 'grafanahost']) -except Exception, e: - print e - usage(1) -for f, v in options: - if f == '-h' or f == '--help': - usage(0) - elif f == '-v' or f == '--verbose': - verbose = True - elif f == '-g' or f == '--grafanahost': - carbonHost = v - else: - print "unknown option : " + f - usage(1) -if carbonHost == '': - print 'grafanahost option is mandatory' - usage(1) -# now parse input and collect statistics -try: - get_wopi_metrics(fileinput.input('-')) -except Exception, e: - print 'Error with collecting metrics:', e - if verbose: - raise - diff --git a/mon/wopi_max_concurrency.py2 b/mon/wopi_max_concurrency.py2 deleted file mode 100755 index 1f74199e..00000000 --- a/mon/wopi_max_concurrency.py2 +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/python -''' -wopi_max_concurrency.py - -A daemon pushing CERNBox WOPI monitoring data to Grafana. -TODO: make it a collectd plugin. References: -https://collectd.org/documentation/manpages/collectd-python.5.shtml -https://blog.dbrgn.ch/2017/3/10/write-a-collectd-python-plugin/ -https://github.com/dbrgn/collectd-python-plugins - -author: Giuseppe.LoPresti@cern.ch -CERN/IT-ST -''' - -import fileinput -import socket -import time -import pickle -import struct -import datetime -import getopt -import sys - -CARBON_TCPPORT = 2004 -carbonHost = '' -verbose = False -prefix = 'cernbox.wopi.' + socket.gethostname().split('.')[0] -epoch = datetime.datetime(1970, 1, 1) - - -def usage(exitCode): - '''prints usage''' - print 'Usage : cat | ' + sys.argv[0] + ' [-h|--help] -g|--grafanahost ' - sys.exit(exitCode) - -def send_metric(data): - '''send data to grafana using the pickle protocol''' - payload = pickle.dumps(data, protocol=2) - header = struct.pack("!L", len(payload)) - message = header + payload - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((carbonHost, CARBON_TCPPORT)) - sock.sendall(message) - sock.close() - -def get_wopi_metrics(data): - '''Parse WOPI usage metrics''' - for line in data: - if data.isfirstline(): - logdate = line.split('T')[0].split('-') # keeps the date until 'T', splits - timestamp = (datetime.datetime(int(logdate[0]), int(logdate[1]), int(logdate[2]), 1, 0, 0) - epoch).total_seconds() + time.altzone - maxconc = 0 - tokens = set() - try: - if 'msg="Lock"' in line and 'INFO' in line and 'result' not in line: - # +1 for this acc. token - l = line.split() - tok = l[-1].split('=')[1] - tokens.add(tok) - if len(tokens) > maxconc: - maxconc += 1 - if 'msg="Unlock"' in line and 'INFO' in line: - # -1 for this acc. token - l = line.split() - tok = l[-1].split('=')[1] - try: - tokens.remove(tok) - except KeyError: - pass - except Exception: - if verbose: - print 'Error occurred at line: %s' % line - raise - - if 'tok' not in locals(): - # the file was empty, nothing to do - return - # prepare data for grafana - output = [] - output.append(( prefix + '.maxconc', (int(timestamp), maxconc) )) - send_metric(output) - if verbose: - print output - - -# first parse options -try: - options, args = getopt.getopt(sys.argv[1:], 'hvg:', ['help', 'verbose', 'grafanahost']) -except Exception, e: - print e - usage(1) -for f, v in options: - if f == '-h' or f == '--help': - usage(0) - elif f == '-v' or f == '--verbose': - verbose = True - elif f == '-g' or f == '--grafanahost': - carbonHost = v - else: - print "unknown option : " + f - usage(1) -if carbonHost == '': - print 'grafanahost option is mandatory' - usage(1) -# now parse input and collect statistics -try: - get_wopi_metrics(fileinput.input('-')) -except Exception, e: - print 'Error with collecting metrics:', e - if verbose: - raise - From f7e261d345df0f1f9a93eb39682f2e0e93af8df0 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 7 Oct 2022 16:04:46 +0200 Subject: [PATCH 036/325] Improved comment --- src/core/commoniface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/commoniface.py b/src/core/commoniface.py index 0b00995f..4ec996dc 100644 --- a/src/core/commoniface.py +++ b/src/core/commoniface.py @@ -19,7 +19,7 @@ # standard error thrown when attempting to overwrite a file/xattr in O_EXCL mode EXCL_ERROR = 'File exists and islock flag requested' -# error thrown when relocking a file and the payload does not match +# error thrown on refreshlock when the payload does not match LOCK_MISMATCH_ERROR = 'Existing lock payload does not match' # standard error thrown when attempting an operation without the required access rights From 73175042f1f8f50c01fbfb728bad1340ce198f92 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 11 Oct 2022 14:40:24 +0200 Subject: [PATCH 037/325] fix duplicate logging entries (#92) * fix duplicate logging entries * conditionally add log handler to flask --- src/wopiserver.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/wopiserver.py b/src/wopiserver.py index f5125a4c..371d20c2 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -107,7 +107,11 @@ def init(cls): fmt='{"time": "%(asctime)s.%(msecs)03d", "host": "' + hostname + '", "level": "%(levelname)s", "process": "%(name)s", %(message)s}', datefmt='%Y-%m-%dT%H:%M:%S')) - cls.app.logger.handlers = [loghandler] + if cls.config.get('general', 'internalserver', fallback='flask') == 'waitress': + cls.log.logger.handlers.clear() + logging.getLogger().handlers = [loghandler] + else: + cls.app.logger.handlers = [loghandler] # load the requested storage layer storage_layer_import(cls.config.get('general', 'storagetype')) # prepare the Flask web app From 5dd87f44c8136d57a15f041c9a31dc7c08cd5d7c Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 11 Oct 2022 14:46:50 +0200 Subject: [PATCH 038/325] Do not expose a FileSharingUrl property if not configured --- src/core/wopi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index b2d845c8..c56cbf2b 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -62,9 +62,9 @@ def checkFileInfo(fileid, acctok): (srv.config.get('general', 'downloadurl'), flask.request.args['access_token']) fmd['BreadcrumbBrandName'] = srv.config.get('general', 'brandingname', fallback=None) fmd['BreadcrumbBrandUrl'] = srv.config.get('general', 'brandingurl', fallback=None) - fmd['FileSharingUrl'] = srv.config.get('general', 'filesharingurl', fallback=None) - if fmd['FileSharingUrl']: - fmd['FileSharingUrl'] = fmd['FileSharingUrl'].replace('', url_quote(acctok['filename'])).replace('', fileid) + fsurl = srv.config.get('general', 'filesharingurl', fallback=None) + if fsurl: + fmd['FileSharingUrl'] = fsurl.replace('', url_quote(acctok['filename'])).replace('', fileid) fmd['OwnerId'] = statInfo['ownerid'] fmd['UserId'] = acctok['wopiuser'] # typically same as OwnerId; different when accessing shared documents fmd['Size'] = statInfo['size'] From c68b47f92efbdb1c4f180ca2a00a7f6c3e38da85 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 11 Oct 2022 14:47:49 +0200 Subject: [PATCH 039/325] Updated contributors --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ffee32ab..b55d0fb9 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Contributors: - Klaas Freitag (@dragotin) - Jörn Friedrich Dreyer (@butonic) - Michael Barz (@micbar) +- Robert Kaussow (@xoxys) Initial revision: December 2016
First production version for CERNBox: September 2017 (presented at [oCCon17](https://occon17.owncloud.org) - [slides](https://www.slideshare.net/giuseppelopresti/collaborative-editing-and-more-in-cernbox))
From 3a14937a9e1edfb381f41821674223082089e887 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 11 Oct 2022 15:26:27 +0200 Subject: [PATCH 040/325] Fix CheckFileInfoSchema test - Collabora Online-specific properties are only exposed to Collabora - ClientUrl property is set regardless of the app, when configured --- src/core/wopi.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index c56cbf2b..d1c7b222 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -79,17 +79,17 @@ def checkFileInfo(fileid, acctok): fmd['SupportsUserInfo'] = False # TODO https://docs.microsoft.com/en-us/openspecs/office_protocols/ms-wopi/371e25ae-e45b-47ab-aec3-9111e962919d # populate app-specific metadata - if acctok['appname'].find('Microsoft') > 0: - # the following is to enable the 'Edit in Word/Excel/PowerPoint' (desktop) action (probably broken) - try: - fmd['ClientUrl'] = srv.config.get('general', 'webdavurl') + '/' + acctok['filename'] - except configparser.NoOptionError: - # if no WebDAV URL is provided, ignore this setting - pass + # the following is to enable the 'Edit in Word/Excel/PowerPoint' (desktop) action (probably broken) + try: + fmd['ClientUrl'] = srv.config.get('general', 'webdavurl') + '/' + acctok['filename'] + except configparser.NoOptionError: + # if no WebDAV URL is provided, ignore this setting + pass # extensions for Collabora Online - fmd['EnableOwnerTermination'] = True - fmd['DisableExport'] = fmd['DisableCopy'] = fmd['DisablePrint'] = acctok['viewmode'] == utils.ViewMode.VIEW_ONLY - # fmd['LastModifiedTime'] = datetime.fromtimestamp(int(statInfo['mtime'])).isoformat() # this currently breaks + if acctok['appname'].find('Collabora') >= 0 or acctok['appname'] == '': + fmd['EnableOwnerTermination'] = True + fmd['DisableExport'] = fmd['DisableCopy'] = fmd['DisablePrint'] = acctok['viewmode'] == utils.ViewMode.VIEW_ONLY + # fmd['LastModifiedTime'] = datetime.fromtimestamp(int(statInfo['mtime'])).isoformat() # this currently breaks res = flask.Response(json.dumps(fmd), mimetype='application/json') # amend sensitive metadata for the logs From 3d4bc724d2c67c25336460fdea814a7e82dd0d1c Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 12 Oct 2022 11:52:41 +0200 Subject: [PATCH 041/325] Reinstated check for external MS Office locks This was a regression from the last refactoring of the setLock code --- src/core/wopi.py | 6 +++++- src/core/wopiutils.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index d1c7b222..4f2a801e 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -176,10 +176,14 @@ def setLock(fileid, reqheaders, acctok): return utils.makeConflictResponse(op, acctok['userid'], None, lock, oldLock, acctok['endpoint'], fn, 'The file was not locked' + ' and got modified' if validateTarget else '') - # now create an "external" lock if required + # now check for and create an "external" lock if required if srv.config.get('general', 'detectexternallocks', fallback='True').upper() == 'TRUE' and \ os.path.splitext(fn)[1] in srv.codetypes: try: + if retrievedLock == 'External': + return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, + acctok['endpoint'], fn, 'The file is locked by ' + lockHolder) + # create a LibreOffice-compatible lock file for interoperability purposes, making sure to # not overwrite any existing or being created lock lockcontent = ',Collaborative Online Editor,%s,%s,WOPIServer;' % \ diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 6a8e01ea..06d3f8f6 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -352,7 +352,7 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, endpoint else: savetime = 0 session = flask.request.headers.get('X-WOPI-SessionId') - if session: + if session and retrievedlock != 'External': if session in srv.conflictsessions['pending']: srv.conflictsessions['pending'][session] += 1 else: From 1bb57f30c6b5b5c72773aa4141f24f94cdead98d Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 12 Oct 2022 12:46:20 +0200 Subject: [PATCH 042/325] xroot: fixed error handling in read/write ops, timeouts are to be handled as general errors --- src/core/xrootiface.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index d0c54cd6..a764f434 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -332,9 +332,6 @@ def readfile(endpoint, filepath, userid, _lockid): rc, _ = f.open(_geturlfor(endpoint) + '/' + homepath + filepath + _eosargs(userid), OpenFlags.READ, timeout=timeout) tend = time.time() - if not f.is_open(): - log.error('msg="Timeout with xroot" op="read" filepath="%s"' % filepath) - raise IOError('Timeout opening file for read') if not rc.ok: # the file could not be opened: check the case of ENOENT and log it as info to keep the logs cleaner if common.ENOENT_MSG in rc.message: @@ -367,9 +364,6 @@ def writefile(endpoint, filepath, userid, content, _lockid, islock=False): rc, _ = f.open(_geturlfor(endpoint) + '/' + homepath + filepath + _eosargs(userid, not islock, size), OpenFlags.NEW if islock else OpenFlags.DELETE, timeout=timeout) tend = time.time() - if not f.is_open(): - log.error('msg="Timeout with xroot" op="write" filepath="%s"' % filepath) - raise IOError('Timeout opening file for write') if not rc.ok: if islock and 'File exists' in rc.message: # racing against an existing file From 6c4d823287b0a62b30a39b0e4af550d6758b6464 Mon Sep 17 00:00:00 2001 From: Javier Ferrer Date: Mon, 17 Oct 2022 09:56:16 +0200 Subject: [PATCH 043/325] Send reva token to codimd (#95) --- src/bridge/__init__.py | 4 ++-- src/bridge/codimd.py | 4 +++- src/wopiserver.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index fe0c4b7b..145df6fd 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -135,7 +135,7 @@ def _gendocid(wopisrc): # The Bridge endpoints start here ############################################################################################################# -def appopen(wopisrc, acctok, appname, viewmode): +def appopen(wopisrc, acctok, appname, viewmode, revatok=None): '''Open a doc by contacting the provided WOPISrc with the given access_token. Returns a (app-url, params{}) pair if successful, raises a FailedOpen exception otherwise''' wopisrc = urlparse.unquote_plus(wopisrc) @@ -206,7 +206,7 @@ def appopen(wopisrc, acctok, appname, viewmode): wopilock = app.loadfromstorage(filemd, wopisrc, acctok, None) redirurl = app.getredirecturl(viewmode, wopisrc, acctok, wopilock['doc'][1:], filemd['BaseFileName'], - filemd['UserFriendlyName']) + filemd['UserFriendlyName'], revatok) except app.AppFailure as e: # this can be raised by loadfromstorage or getredirecturl usermsg = str(e) if str(e) else 'Unable to load the app, please try again later or contact support' diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 3b710a8e..8bd02f29 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -53,7 +53,7 @@ def init(_appurl, _appinturl, _apikey): raise AppFailure -def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname): +def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname, revatok=None): '''Return a valid URL to the app for the given WOPI context''' if viewmode in (utils.ViewMode.READ_WRITE, utils.ViewMode.PREVIEW): mode = 'view' if viewmode == utils.ViewMode.PREVIEW else 'both' @@ -63,6 +63,8 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname): 'disableEmbedding': ('%s' % (os.path.splitext(filename)[1] != '.zmd')).lower(), 'displayName': displayname, } + if revatok: + params['revaToken'] = revatok return f'{appexturl}/{docid}?{mode}&{urlparse.urlencode(params)}' # read-only mode: first check if we have a CodiMD redirection diff --git a/src/wopiserver.py b/src/wopiserver.py index 371d20c2..aa87dc6b 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -354,7 +354,8 @@ def iopOpenInApp(): res = {} if bridge.issupported(appname): try: - res['app-url'], res['form-parameters'] = bridge.appopen(utils.generateWopiSrc(inode), acctok, appname, viewmode) + res['app-url'], res['form-parameters'] = bridge.appopen(utils.generateWopiSrc(inode), acctok, appname, viewmode, + usertoken) except bridge.FailedOpen as foe: return foe.msg, foe.statuscode else: From 913e9e45985a0e598da11be8cb8ae98e47790f50 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 11 Oct 2022 20:15:52 +0200 Subject: [PATCH 044/325] Implemented configurable host URLs --- src/core/commoniface.py | 2 +- src/core/wopi.py | 32 +++++++++++++++++++++++++------- src/core/wopiutils.py | 6 ++++++ wopiserver.conf | 14 ++++++++++++-- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/core/commoniface.py b/src/core/commoniface.py index 4ec996dc..b6857218 100644 --- a/src/core/commoniface.py +++ b/src/core/commoniface.py @@ -81,7 +81,7 @@ def retrieverevalock(rawlock): def encodeinode(endpoint, inode): '''Encodes a given endpoint and inode to be used as a safe WOPISrc: endpoint is assumed to already be URL safe''' - return endpoint + '-' + urlsafe_b64encode(inode.encode()).decode() + return endpoint + '!' + urlsafe_b64encode(inode.encode()).decode() def validatelock(filepath, appname, oldlock, oldvalue, op, log): diff --git a/src/core/wopi.py b/src/core/wopi.py index 4f2a801e..41c10506 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -38,8 +38,19 @@ def checkFileInfo(fileid, acctok): fmd['BaseFileName'] = fmd['BreadcrumbDocName'] = os.path.basename(acctok['filename']) wopiSrc = 'WOPISrc=%s&access_token=%s' % (utils.generateWopiSrc(fileid, acctok['appname'] == srv.proxiedappname), flask.request.args['access_token']) - fmd['HostViewUrl'] = '%s%s%s' % (acctok['appviewurl'], '&' if '?' in acctok['appviewurl'] else '?', wopiSrc) - fmd['HostEditUrl'] = '%s%s%s' % (acctok['appediturl'], '&' if '?' in acctok['appediturl'] else '?', wopiSrc) + hosteurl = srv.config.get('general', 'hostediturl', fallback=None) + if hosteurl: + fmd['HostEditUrl'] = utils.generateUrlFromTemplate(hosteurl, acctok, fileid) + else: + fmd['HostEditUrl'] = '%s%s%s' % (acctok['appediturl'], '&' if '?' in acctok['appediturl'] else '?', wopiSrc) + hostvurl = srv.config.get('general', 'hostviewurl', fallback=None) + if hostvurl: + fmd['HostViewUrl'] = utils.generateUrlFromTemplate(hostvurl, acctok, fileid) + else: + fmd['HostViewUrl'] = '%s%s%s' % (acctok['appviewurl'], '&' if '?' in acctok['appviewurl'] else '?', wopiSrc) + fsurl = srv.config.get('general', 'filesharingurl', fallback=None) + if fsurl: + fmd['FileSharingUrl'] = utils.generateUrlFromTemplate(fsurl, acctok, fileid) furl = acctok['folderurl'] fmd['BreadcrumbFolderUrl'] = furl if furl != '/' else srv.wopiurl # the WOPI URL is a placeholder if acctok['username'] == '': @@ -62,9 +73,6 @@ def checkFileInfo(fileid, acctok): (srv.config.get('general', 'downloadurl'), flask.request.args['access_token']) fmd['BreadcrumbBrandName'] = srv.config.get('general', 'brandingname', fallback=None) fmd['BreadcrumbBrandUrl'] = srv.config.get('general', 'brandingurl', fallback=None) - fsurl = srv.config.get('general', 'filesharingurl', fallback=None) - if fsurl: - fmd['FileSharingUrl'] = fsurl.replace('', url_quote(acctok['filename'])).replace('', fileid) fmd['OwnerId'] = statInfo['ownerid'] fmd['UserId'] = acctok['wopiuser'] # typically same as OwnerId; different when accessing shared documents fmd['Size'] = statInfo['size'] @@ -416,8 +424,18 @@ def putRelative(fileid, reqheaders, acctok): putrelmd['Name'] = os.path.basename(targetName) newwopisrc = '%s&access_token=%s' % (utils.generateWopiSrc(inode, acctok['appname'] == srv.proxiedappname), newacctok) putrelmd['Url'] = url_unquote(newwopisrc).replace('&access_token', '?access_token') - putrelmd['HostEditUrl'] = '%s%s%s' % (acctok['appediturl'], '&' if '?' in acctok['appediturl'] else '?', newwopisrc) - putrelmd['HostViewUrl'] = '%s%s%s' % (acctok['appviewurl'], '&' if '?' in acctok['appediturl'] else '?', newwopisrc) + hosteurl = srv.config.get('general', 'hostediturl', fallback=None) + if hosteurl: + putrelmd['HostEditUrl'] = utils.generateUrlFromTemplate(hosteurl, + {'appname': acctok['appname'], 'filename': targetName}, inode) + else: + putrelmd['HostEditUrl'] = '%s%s%s' % (acctok['appediturl'], '&' if '?' in acctok['appediturl'] else '?', newwopisrc) + hostvurl = srv.config.get('general', 'hostviewurl', fallback=None) + if hostvurl: + putrelmd['HostViewUrl'] = utils.generateUrlFromTemplate(hostvurl, + {'appname': acctok['appname'], 'filename': targetName}, inode) + else: + putrelmd['HostViewUrl'] = '%s%s%s' % (acctok['appviewurl'], '&' if '?' in acctok['appviewurl'] else '?', newwopisrc) resp = flask.Response(json.dumps(putrelmd), mimetype='application/json') putrelmd['Url'] = putrelmd['HostEditUrl'] = putrelmd['HostViewUrl'] = '_redacted_' log.info('msg="PutRelative response" token="%s" metadata="%s"' % (newacctok[-20:], putrelmd)) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 06d3f8f6..1991af32 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -146,6 +146,12 @@ def generateWopiSrc(fileid, proxy=False): return url_quote_plus('%s/wopi/files/%s' % (srv.wopiproxy, fileid)).replace('-', '%2D') +def generateUrlFromTemplate(url, acctok, fileid): + '''One-liner to parse an URL template and return it with actualised placeholders''' + return url.replace('', url_quote_plus(acctok['filename'])). \ + replace('', fileid).replace('', acctok['appname']) + + def getLibreOfficeLockName(filename): '''Returns the filename of a LibreOffice-compatible lock file. This enables interoperability between Online and Desktop applications''' diff --git a/wopiserver.conf b/wopiserver.conf index 26b51f82..b1acd908 100644 --- a/wopiserver.conf +++ b/wopiserver.conf @@ -44,9 +44,19 @@ loghandler = file # Optional URL to display a file sharing dialog. This enables # a 'Share' button within the application. The URL may contain -# either the `` or `` placeholders, which are +# the ``, ``, and `` placeholders, which are # dynamically replaced with actual values for the opened file. -#filesharingurl = https://your-efss-server.org/fileshare?filepath=&resource= +#filesharingurl = https://your-efss-server.org/fileshare?filepath=&app=&resource= + +# URLs for the pages that embed the application in edit mode and +# preview mode. By default, the appediturl and appviewurl are used, +# but it is recommended to configure here a URL that displays apps +# within an iframe on your EFSS. +# Placeholders ``, ``, and `` are dynamically +# replaced similarly to the above. The suggested example reflects +# the ownCloud web implementation. +#hostediturl = https://your-efss-server.org/external/spaces?app=&fileId= +#hostviewurl = https://your-efss-server.org/external/spaces?app=&fileId=&viewmode=VIEW_MODE_PREVIEW # Optional URL prefix for WebDAV access to the files. This enables # a 'Edit in Desktop client' action on Windows-based clients From c780546d0f46734a032a966c4114cf4ea10da90c Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 11 Oct 2022 20:45:43 +0200 Subject: [PATCH 045/325] Fixed fileid on host URLs in case the wopiproxy is configured This implied adding the original fileid to the WOPI access token, and altering the internal format of inodes to be compatible with the web frontend. --- src/core/commoniface.py | 9 ++++++++- src/core/wopi.py | 28 +++++++++++++++++----------- src/core/wopiutils.py | 16 ++++++++-------- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/core/commoniface.py b/src/core/commoniface.py index b6857218..9a4a7ce0 100644 --- a/src/core/commoniface.py +++ b/src/core/commoniface.py @@ -80,10 +80,17 @@ def retrieverevalock(rawlock): def encodeinode(endpoint, inode): - '''Encodes a given endpoint and inode to be used as a safe WOPISrc: endpoint is assumed to already be URL safe''' + '''Encodes a given endpoint and inode to be used as a safe WOPISrc: endpoint is assumed to already be URL safe. + Note that the separator is chosen to be `!` for compatibility with the ownCloud Web frontend.''' return endpoint + '!' + urlsafe_b64encode(inode.encode()).decode() +def decodeinode(inode): + '''Decodes an inode obtained from encodeinode()''' + e, f = inode.split('!') + return e, urlsafe_b64decode(f.encode()).decode() + + def validatelock(filepath, appname, oldlock, oldvalue, op, log): '''Common logic for validating locks in the xrootd and local storage interfaces. Duplicates some logic implemented in Reva for the cs3 storage interface''' diff --git a/src/core/wopi.py b/src/core/wopi.py index 41c10506..12299a23 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -13,7 +13,6 @@ import http.client from datetime import datetime from urllib.parse import unquote_plus as url_unquote -from urllib.parse import quote_plus as url_quote from more_itertools import peekable import flask import core.wopiutils as utils @@ -40,17 +39,17 @@ def checkFileInfo(fileid, acctok): flask.request.args['access_token']) hosteurl = srv.config.get('general', 'hostediturl', fallback=None) if hosteurl: - fmd['HostEditUrl'] = utils.generateUrlFromTemplate(hosteurl, acctok, fileid) + fmd['HostEditUrl'] = utils.generateUrlFromTemplate(hosteurl, acctok) else: fmd['HostEditUrl'] = '%s%s%s' % (acctok['appediturl'], '&' if '?' in acctok['appediturl'] else '?', wopiSrc) hostvurl = srv.config.get('general', 'hostviewurl', fallback=None) if hostvurl: - fmd['HostViewUrl'] = utils.generateUrlFromTemplate(hostvurl, acctok, fileid) + fmd['HostViewUrl'] = utils.generateUrlFromTemplate(hostvurl, acctok) else: fmd['HostViewUrl'] = '%s%s%s' % (acctok['appviewurl'], '&' if '?' in acctok['appviewurl'] else '?', wopiSrc) fsurl = srv.config.get('general', 'filesharingurl', fallback=None) if fsurl: - fmd['FileSharingUrl'] = utils.generateUrlFromTemplate(fsurl, acctok, fileid) + fmd['FileSharingUrl'] = utils.generateUrlFromTemplate(fsurl, acctok) furl = acctok['folderurl'] fmd['BreadcrumbFolderUrl'] = furl if furl != '/' else srv.wopiurl # the WOPI URL is a placeholder if acctok['username'] == '': @@ -407,6 +406,8 @@ def putRelative(fileid, reqheaders, acctok): # either way, we now have a targetName to save the file: attempt to do so try: utils.storeWopiFile(acctok, None, utils.LASTSAVETIMEKEY, targetName) + newstat = st.statx(acctok['endpoint'], targetName, acctok['userid']) + _, newfileid = common.decodeinode(newstat['inode']) except IOError as e: utils.storeForRecovery(flask.request.get_data(), acctok['username'], targetName, flask.request.args['access_token'][-20:], e) @@ -420,20 +421,25 @@ def putRelative(fileid, reqheaders, acctok): acctok['folderurl'], acctok['endpoint'], (acctok['appname'], acctok['appediturl'], acctok['appviewurl'])) # prepare and send the response as JSON - putrelmd = {} - putrelmd['Name'] = os.path.basename(targetName) + mdforhosturls = { + 'appname': acctok['appname'], + 'filename': targetName, + 'endpoint': acctok['endpoint'], + 'fileid': newfileid, + } newwopisrc = '%s&access_token=%s' % (utils.generateWopiSrc(inode, acctok['appname'] == srv.proxiedappname), newacctok) - putrelmd['Url'] = url_unquote(newwopisrc).replace('&access_token', '?access_token') + putrelmd = { + 'Name': os.path.basename(targetName), + 'Url': url_unquote(newwopisrc).replace('&access_token', '?access_token'), + } hosteurl = srv.config.get('general', 'hostediturl', fallback=None) if hosteurl: - putrelmd['HostEditUrl'] = utils.generateUrlFromTemplate(hosteurl, - {'appname': acctok['appname'], 'filename': targetName}, inode) + putrelmd['HostEditUrl'] = utils.generateUrlFromTemplate(hosteurl, mdforhosturls) else: putrelmd['HostEditUrl'] = '%s%s%s' % (acctok['appediturl'], '&' if '?' in acctok['appediturl'] else '?', newwopisrc) hostvurl = srv.config.get('general', 'hostviewurl', fallback=None) if hostvurl: - putrelmd['HostViewUrl'] = utils.generateUrlFromTemplate(hostvurl, - {'appname': acctok['appname'], 'filename': targetName}, inode) + putrelmd['HostViewUrl'] = utils.generateUrlFromTemplate(hostvurl, mdforhosturls) else: putrelmd['HostViewUrl'] = '%s%s%s' % (acctok['appviewurl'], '&' if '?' in acctok['appviewurl'] else '?', newwopisrc) resp = flask.Response(json.dumps(putrelmd), mimetype='application/json') diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 1991af32..46bed2be 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -146,10 +146,11 @@ def generateWopiSrc(fileid, proxy=False): return url_quote_plus('%s/wopi/files/%s' % (srv.wopiproxy, fileid)).replace('-', '%2D') -def generateUrlFromTemplate(url, acctok, fileid): - '''One-liner to parse an URL template and return it with actualised placeholders''' +def generateUrlFromTemplate(url, acctok): + '''One-liner to parse an URL template and return it with actualised placeholders. See also common.encodeinode()''' return url.replace('', url_quote_plus(acctok['filename'])). \ - replace('', fileid).replace('', acctok['appname']) + replace('', acctok['endpoint'] + '!' + acctok['fileid']). \ + replace('', acctok['appname']) def getLibreOfficeLockName(filename): @@ -182,8 +183,8 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app log.debug('msg="Generating token" userid="%s" fileid="%s" endpoint="%s" app="%s"' % (userid[-20:], fileid, endpoint, appname)) try: - # stat the file to check for existence and get a version-invariant inode and modification time: - # the inode serves as fileid (and must not change across save operations), the mtime is used for version information. + # stat the file to check for existence and get a version-invariant inode: + # the inode serves as fileid (and must not change across save operations) statinfo = st.statx(endpoint, fileid, userid) except IOError as e: log.info('msg="Requested file not found or not a file" fileid="%s" error="%s"' % (fileid, e)) @@ -208,8 +209,8 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app # does not set appname when the app is not proxied, so we optimistically assume it's Collabora and let it go) log.info('msg="Forcing read-only access to ODF file" filename="%s"' % statinfo['filepath']) viewmode = ViewMode.READ_ONLY - acctok = jwt.encode({'userid': userid, 'wopiuser': wopiuser, 'filename': statinfo['filepath'], 'username': username, - 'viewmode': viewmode.value, 'folderurl': folderurl, 'endpoint': endpoint, + acctok = jwt.encode({'userid': userid, 'wopiuser': wopiuser, 'filename': statinfo['filepath'], 'fileid': fileid, + 'username': username, 'viewmode': viewmode.value, 'folderurl': folderurl, 'endpoint': endpoint, 'appname': appname, 'appediturl': appediturl, 'appviewurl': appviewurl, 'exp': exptime, 'iss': 'cs3org:wopiserver:%s' % WOPIVER}, # standard claims srv.wopisecret, algorithm='HS256') @@ -218,7 +219,6 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app (userid[-20:], wopiuser if wopiuser != userid else username, viewmode, endpoint, statinfo['filepath'], statinfo['inode'], statinfo['mtime'], folderurl, appname, exptime, acctok[-20:])) - # return the inode == fileid, the filepath and the access token return statinfo['inode'], acctok, viewmode From c8ef3f29fb944c89ff316a4a9639b9659babc3f7 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 12 Oct 2022 11:32:04 +0200 Subject: [PATCH 046/325] Decoupled the fileId separator from the web frontend implementation and made it part of the configuration --- src/core/commoniface.py | 3 ++- src/core/wopiutils.py | 5 +++-- wopiserver.conf | 17 +++++++++-------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/core/commoniface.py b/src/core/commoniface.py index 9a4a7ce0..6eeffaea 100644 --- a/src/core/commoniface.py +++ b/src/core/commoniface.py @@ -81,7 +81,8 @@ def retrieverevalock(rawlock): def encodeinode(endpoint, inode): '''Encodes a given endpoint and inode to be used as a safe WOPISrc: endpoint is assumed to already be URL safe. - Note that the separator is chosen to be `!` for compatibility with the ownCloud Web frontend.''' + Note that the separator is chosen to be `!` (similar to how the web frontend is implemented) to allow the inverse + operation, assuming that `endpoint` does not contain any `!` characters.''' return endpoint + '!' + urlsafe_b64encode(inode.encode()).decode() diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 46bed2be..897832f7 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -147,9 +147,10 @@ def generateWopiSrc(fileid, proxy=False): def generateUrlFromTemplate(url, acctok): - '''One-liner to parse an URL template and return it with actualised placeholders. See also common.encodeinode()''' + '''One-liner to parse an URL template and return it with actualised placeholders. See also common.encodeinode().''' return url.replace('', url_quote_plus(acctok['filename'])). \ - replace('', acctok['endpoint'] + '!' + acctok['fileid']). \ + replace('', acctok['endpoint']). \ + replace('', acctok['fileid']). \ replace('', acctok['appname']) diff --git a/wopiserver.conf b/wopiserver.conf index b1acd908..ce93254b 100644 --- a/wopiserver.conf +++ b/wopiserver.conf @@ -44,19 +44,20 @@ loghandler = file # Optional URL to display a file sharing dialog. This enables # a 'Share' button within the application. The URL may contain -# the ``, ``, and `` placeholders, which are -# dynamically replaced with actual values for the opened file. -#filesharingurl = https://your-efss-server.org/fileshare?filepath=&app=&resource= +# any of the ``, ``, ``, and `` +# placeholders, which are dynamically replaced with actual values +# for the opened file. +#filesharingurl = https://your-efss-server.org/fileshare?filepath=&app=&fileId=! # URLs for the pages that embed the application in edit mode and # preview mode. By default, the appediturl and appviewurl are used, # but it is recommended to configure here a URL that displays apps # within an iframe on your EFSS. -# Placeholders ``, ``, and `` are dynamically -# replaced similarly to the above. The suggested example reflects -# the ownCloud web implementation. -#hostediturl = https://your-efss-server.org/external/spaces?app=&fileId= -#hostviewurl = https://your-efss-server.org/external/spaces?app=&fileId=&viewmode=VIEW_MODE_PREVIEW +# Placeholders ``, ``, ``, and `` are +# dynamically replaced similarly to the above. The suggested example +# reflects the ownCloud web implementation. +#hostediturl = https://your-efss-server.org/external/spaces?app=&fileId=! +#hostviewurl = https://your-efss-server.org/external/spaces?app=&fileId=!&viewmode=VIEW_MODE_PREVIEW # Optional URL prefix for WebDAV access to the files. This enables # a 'Edit in Desktop client' action on Windows-based clients From 1e4b14090b2108e621b1b970df63e020afaee19c Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 17 Oct 2022 09:58:51 +0200 Subject: [PATCH 047/325] CodiMD: removed commented code The direct link to slide mode is going to be added in CodiMD, no need to hack around on the wopiserver for this --- src/bridge/codimd.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 8bd02f29..1633b55d 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -71,9 +71,6 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname, reva res = requests.head(appurl + '/' + docid, verify=sslverify) if res.status_code == http.client.FOUND: return '%s/s/%s' % (appexturl, urlparse.urlsplit(res.next.url).path.split('/')[-1]) - # we used to redirect to publish mode or normal view to quickly jump in slide mode depending on the content, - # but this was based on a bad side effect - here it would require to add: - # ('/publish' if not _isslides(content) else '') before the '?' return f'{appexturl}/{docid}/publish' @@ -116,11 +113,6 @@ def _unzipattachments(inputbuf): return mddoc -# def _isslides(doc): -# '''Heuristically look for signatures of slides in the header of a md document''' -# return doc[:9].decode() == '---\ntitle' or doc[:8].decode() == '---\ntype' or doc[:16].decode() == '---\nslideOptions' - - def _fetchfromcodimd(wopilock, acctok): '''Fetch a given document from from CodiMD, raise AppFailure in case of errors''' try: From b56fba1fd055ef95c5ce2dbf9bc714e0f07f287d Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 17 Oct 2022 10:08:18 +0200 Subject: [PATCH 048/325] Prepare release --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fbb5a09..ed8db797 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ ## Changelog for the WOPI server +### Mon Oct 17 2022 - v9.2.0 +- Added option to use file or stream handler for logging (#91) +- Introduced configurable hostURLs for CheckFileInfo (#93) +- Fixed duplicate log entries (#92) +- CodiMD: added support for direct storage access via + the ownCloud file picker (#95) +- Fixed check for external locks +- Further fixes to improve coverage of the WOPI validator tests + ### Wed Oct 5 2022 - v9.1.0 - Introduced support for PREVIEW mode (#82) - Improved UnlockAndRelock logic (#85, #87) From 2de542bf67aee9f7b72d0ef9927ab4fa05cbf8ff Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 17 Oct 2022 14:29:23 +0200 Subject: [PATCH 049/325] CodiMD: removed dead code --- src/bridge/codimd.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 1633b55d..25f1cc49 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -67,10 +67,7 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname, reva params['revaToken'] = revatok return f'{appexturl}/{docid}?{mode}&{urlparse.urlencode(params)}' - # read-only mode: first check if we have a CodiMD redirection - res = requests.head(appurl + '/' + docid, verify=sslverify) - if res.status_code == http.client.FOUND: - return '%s/s/%s' % (appexturl, urlparse.urlsplit(res.next.url).path.split('/')[-1]) + # read-only mode: use the publish view of CodiMD return f'{appexturl}/{docid}/publish' From d588939e5ff044c5658ae9946f748e73a588c8ee Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 17 Oct 2022 15:29:10 +0200 Subject: [PATCH 050/325] Fixed example in config file --- wopiserver.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wopiserver.conf b/wopiserver.conf index ce93254b..85e93eab 100644 --- a/wopiserver.conf +++ b/wopiserver.conf @@ -56,8 +56,8 @@ loghandler = file # Placeholders ``, ``, ``, and `` are # dynamically replaced similarly to the above. The suggested example # reflects the ownCloud web implementation. -#hostediturl = https://your-efss-server.org/external/spaces?app=&fileId=! -#hostviewurl = https://your-efss-server.org/external/spaces?app=&fileId=!&viewmode=VIEW_MODE_PREVIEW +#hostediturl = https://your-efss-server.org/external?app=&fileId=! +#hostviewurl = https://your-efss-server.org/external?app=&fileId=!&viewmode=VIEW_MODE_PREVIEW # Optional URL prefix for WebDAV access to the files. This enables # a 'Edit in Desktop client' action on Windows-based clients From 2938df79d8f3948895d7f3cfb5d84f1bf584b8c3 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 18 Oct 2022 08:07:51 +0200 Subject: [PATCH 051/325] Minor refactoring --- src/core/wopiutils.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 897832f7..6d3f0d75 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -429,15 +429,14 @@ def storeAfterConflict(acctok, retrievedlock, lock, reason): storeForRecovery(flask.request.get_data(), acctok['username'], newname, flask.request.args['access_token'][-20:], dorecovery) # conflict file was stored on recovery space, tell user (but reason is advisory...) - return makeConflictResponse('PUTFILE', acctok['userid'], retrievedlock, lock, 'NA', - acctok['endpoint'], acctok['filename'], - reason + ', please contact support to recover it') + reason += ', please contact support to recover it' + else: + # otherwise, conflict file was saved to user space + reason += ', conflict copy created' - # otherwise, conflict file was saved to user space but we still use a CONFLICT response - # as it is better handled by the app to signal the issue to the user + # use a CONFLICT response as it is better handled by the app to signal the issue to the user return makeConflictResponse('PUTFILE', acctok['userid'], retrievedlock, lock, 'NA', - acctok['endpoint'], acctok['filename'], - reason + ', conflict copy created') + acctok['endpoint'], acctok['filename'], reason) def storeForRecovery(content, username, filename, acctokforlog, exception): From c7fcf65f414c7d9f03dc2bc8c62d695714815e07 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 18 Oct 2022 08:09:24 +0200 Subject: [PATCH 052/325] Rationalized accounting logic and removed dead code --- src/core/wopi.py | 32 ++------------------------------ src/core/wopiutils.py | 42 ++++++++++++++++++++++++++++-------------- 2 files changed, 30 insertions(+), 44 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 12299a23..0eb499b1 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -241,16 +241,8 @@ def setLock(fileid, reqheaders, acctok): # not fatal, but will generate a conflict file later on, so log a warning log.warning('msg="Unable to set lastwritetime xattr" lockop="%s" user="%s" filename="%s" token="%s" reason="%s"' % (op.title(), acctok['userid'][-20:], fn, flask.request.args['access_token'][-20:], e)) - # also, keep track of files that have been opened for write: this is for statistical purposes only - # (cf. the GetLock WOPI call and the /wopi/iop/open/list action) - if fn not in srv.openfiles: - srv.openfiles[fn] = (time.asctime(), set([acctok['username']])) - else: - # the file was already opened but without lock: this happens on new files (cf. editnew action), just log - log.info('msg="First lock for new file" lockop="%s" user="%s" filename="%s" token="%s"' % - (op.title(), acctok['userid'][-20:], fn, flask.request.args['access_token'][-20:])) - return utils.makeLockSuccessResponse(op, fn, lock, 'v%s' % statInfo['etag']) + return utils.makeLockSuccessResponse(op, acctok, lock, 'v%s' % statInfo['etag']) except IOError as e: if common.EXCL_ERROR in str(e): @@ -270,7 +262,7 @@ def setLock(fileid, reqheaders, acctok): try: st.refreshlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(lock), utils.encodeLock(oldLock) if oldLock else None) - return utils.makeLockSuccessResponse(op, fn, lock, 'v%s' % statInfo['etag']) + return utils.makeLockSuccessResponse(op, acctok, lock, 'v%s' % statInfo['etag']) except IOError as rle: # this is unexpected now log.error('msg="Failed to refresh lock" lockop="%s" filename="%s" token="%s" lock="%s" error="%s"' % @@ -288,23 +280,6 @@ def getLock(fileid, _reqheaders_unused, acctok): lock, _ = utils.retrieveWopiLock(fileid, 'GETLOCK', '', acctok) resp.status_code = http.client.OK if lock else http.client.NOT_FOUND resp.headers['X-WOPI-Lock'] = lock if lock else '' - # for statistical purposes, check whether a lock exists and update internal bookkeeping - if lock and lock != 'External': - try: - # the file was already opened for write, check whether this is a new user - if not acctok['username'] in srv.openfiles[acctok['filename']][1]: - # yes it's a new user - srv.openfiles[acctok['filename']][1].add(acctok['username']) - if len(srv.openfiles[acctok['filename']][1]) > 1: - # for later monitoring, explicitly log that this file is being edited by at least two users - log.info('msg="Collaborative editing detected" filename="%s" token="%s" users="%s"' % - (acctok['filename'], flask.request.args['access_token'][-20:], - list(srv.openfiles[acctok['filename']][1]))) - except KeyError: - # existing lock but missing srv.openfiles[acctok['filename']] ? - log.warning('msg="Repopulating missing metadata" filename="%s" token="%s" friendlyname="%s"' % - (acctok['filename'], flask.request.args['access_token'][-20:], acctok['username'])) - srv.openfiles[acctok['filename']] = (time.asctime(), set([acctok['username']])) return resp @@ -521,9 +496,6 @@ def _createNewFile(fileid, acctok): utils.storeWopiFile(acctok, None, utils.LASTSAVETIMEKEY) log.info('msg="File stored successfully" action="editnew" user="%s" filename="%s" token="%s"' % (acctok['userid'][-20:], acctok['filename'], flask.request.args['access_token'][-20:])) - # and we keep track of it as an open file with timestamp = Epoch, despite not having any lock yet. - # XXX this is to work around an issue with concurrent editing of newly created files (cf. iopOpen) - srv.openfiles[acctok['filename']] = ('0', set([acctok['username']])) return 'OK', http.client.OK except IOError as e: utils.storeForRecovery(flask.request.get_data(), acctok['username'], acctok['filename'], diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 6d3f0d75..2112c97b 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -353,17 +353,15 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, endpoint reason = {'message': reason} resp.headers['X-WOPI-LockFailureReason'] = reason['message'] resp.data = json.dumps(reason) + + session = flask.request.headers.get('X-WOPI-SessionId') + if session and retrievedlock != 'External' and session not in srv.conflictsessions['pending']: + srv.conflictsessions['pending'][session] = (time.asctime(), retrievedlock) savetime = st.getxattr(endpoint, filename, user, LASTSAVETIMEKEY) if savetime: savetime = int(savetime) else: savetime = 0 - session = flask.request.headers.get('X-WOPI-SessionId') - if session and retrievedlock != 'External': - if session in srv.conflictsessions['pending']: - srv.conflictsessions['pending'][session] += 1 - else: - srv.conflictsessions['pending'][session] = 1 log.warning('msg="Returning conflict" lockop="%s" user="%s" filename="%s" token="%s" sessionId="%s" lock="%s" ' 'oldlock="%s" retrievedlock="%s" fileage="%1.1f" reason="%s"' % (operation.title(), user, filename, flask.request.args['access_token'][-20:], @@ -372,15 +370,23 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, endpoint return resp -def makeLockSuccessResponse(operation, filename, lock, version): +def makeLockSuccessResponse(operation, acctok, lock, version): '''Generates and logs an HTTP 200 response with appropriate headers for Lock/RefreshLock operations''' session = flask.request.headers.get('X-WOPI-SessionId') if session in srv.conflictsessions['pending']: - counter = srv.conflictsessions['pending'].pop(session) - srv.conflictsessions['resolved'][session] = counter + srv.conflictsessions['pending'].pop(session) + srv.conflictsessions['resolved'][session] = time.asctime() + try: + # keep some accounting of the open files + if not session: + session = acctok['username'] + if session not in srv.openfiles[acctok['filename']][1]: + srv.openfiles[acctok['filename']][1].add(session) + except KeyError: + srv.openfiles[acctok['filename']] = (time.asctime(), set(session)) log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s"' % - (operation.title(), filename, flask.request.args['access_token'][-20:], session, lock)) + (operation.title(), acctok['filename'], flask.request.args['access_token'][-20:], session, lock)) resp = flask.Response() resp.status_code = http.client.OK resp.headers['X-WOPI-ItemVersion'] = version @@ -390,13 +396,21 @@ def makeLockSuccessResponse(operation, filename, lock, version): def storeWopiFile(acctok, retrievedlock, xakey, targetname=''): '''Saves a file from an HTTP request to the given target filename (defaulting to the access token's one), and stores the save time as an xattr. Throws IOError in case of any failure''' + if not targetname: + targetname = acctok['filename'] session = flask.request.headers.get('X-WOPI-SessionId') if session in srv.conflictsessions['pending']: - counter = srv.conflictsessions['pending'].pop(session) - srv.conflictsessions['resolved'][session] = counter + srv.conflictsessions['pending'].pop(session) + srv.conflictsessions['resolved'][session] = time.asctime() + try: + # keep some accounting of the open files + if not session: + session = acctok['username'] + if session not in srv.openfiles[targetname][1]: + srv.openfiles[targetname][1].add(session) + except KeyError: + srv.openfiles[targetname] = (time.asctime(), set(session)) - if not targetname: - targetname = acctok['filename'] st.writefile(acctok['endpoint'], targetname, acctok['userid'], flask.request.get_data(), encodeLock(retrievedlock)) # save the current time for later conflict checking: this is never older than the mtime of the file st.setxattr(acctok['endpoint'], targetname, acctok['userid'], xakey, int(time.time()), encodeLock(retrievedlock)) From 4715a087a83a7fc41505021611a4064891179755 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 18 Oct 2022 09:22:52 +0200 Subject: [PATCH 053/325] Nicer format for JSON-ification --- src/core/wopiutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 2112c97b..55db4616 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -356,7 +356,7 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, endpoint session = flask.request.headers.get('X-WOPI-SessionId') if session and retrievedlock != 'External' and session not in srv.conflictsessions['pending']: - srv.conflictsessions['pending'][session] = (time.asctime(), retrievedlock) + srv.conflictsessions['pending'][session] = {'time': time.asctime(), 'held': retrievedlock} savetime = st.getxattr(endpoint, filename, user, LASTSAVETIMEKEY) if savetime: savetime = int(savetime) From 0f17bc572e0c17ae5644c61e93e8cee8df7b6438 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 18 Oct 2022 09:49:54 +0200 Subject: [PATCH 054/325] Further fix on the refactoring of the accounting --- src/core/wopiutils.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 55db4616..d174f03c 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -376,14 +376,13 @@ def makeLockSuccessResponse(operation, acctok, lock, version): if session in srv.conflictsessions['pending']: srv.conflictsessions['pending'].pop(session) srv.conflictsessions['resolved'][session] = time.asctime() - try: - # keep some accounting of the open files - if not session: - session = acctok['username'] - if session not in srv.openfiles[acctok['filename']][1]: - srv.openfiles[acctok['filename']][1].add(session) - except KeyError: - srv.openfiles[acctok['filename']] = (time.asctime(), set(session)) + # keep some accounting of the open files + if not session: + session = acctok['username'] + if acctok['filename'] not in srv.openfiles: + srv.openfiles[acctok['filename']] = (time.asctime(), set()) + if session not in srv.openfiles[acctok['filename']][1]: + srv.openfiles[acctok['filename']][1].add(session) log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s"' % (operation.title(), acctok['filename'], flask.request.args['access_token'][-20:], session, lock)) @@ -402,14 +401,13 @@ def storeWopiFile(acctok, retrievedlock, xakey, targetname=''): if session in srv.conflictsessions['pending']: srv.conflictsessions['pending'].pop(session) srv.conflictsessions['resolved'][session] = time.asctime() - try: - # keep some accounting of the open files - if not session: - session = acctok['username'] - if session not in srv.openfiles[targetname][1]: - srv.openfiles[targetname][1].add(session) - except KeyError: - srv.openfiles[targetname] = (time.asctime(), set(session)) + # keep some accounting of the open files + if not session: + session = acctok['username'] + if targetname not in srv.openfiles: + srv.openfiles[targetname] = (time.asctime(), set()) + if session not in srv.openfiles[targetname][1]: + srv.openfiles[targetname][1].add(session) st.writefile(acctok['endpoint'], targetname, acctok['userid'], flask.request.get_data(), encodeLock(retrievedlock)) # save the current time for later conflict checking: this is never older than the mtime of the file From fff51ef56c85b540adac5028b742d3cc2dd68d32 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 20 Oct 2022 07:56:10 +0200 Subject: [PATCH 055/325] Do not URL-encode paths in templated URLs They are typically used as part of the URL itself, not as query parameters. If paths are to be hidden, they can just be omitted in the templated configuration so to rely on fileid/endpoint only. --- src/core/wopiutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index d174f03c..4ea3feea 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -147,8 +147,8 @@ def generateWopiSrc(fileid, proxy=False): def generateUrlFromTemplate(url, acctok): - '''One-liner to parse an URL template and return it with actualised placeholders. See also common.encodeinode().''' - return url.replace('', url_quote_plus(acctok['filename'])). \ + '''One-liner to parse an URL template and return it with actualised placeholders''' + return url.replace('', acctok['filename']). \ replace('', acctok['endpoint']). \ replace('', acctok['fileid']). \ replace('', acctok['appname']) From 172b77b496ba86c1a2b237ea8d0e8ab98003e576 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 22 Oct 2022 15:37:04 +0200 Subject: [PATCH 056/325] xroot: fixed case of None returned by stat --- src/core/xrootiface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index a764f434..bfe7736d 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -213,7 +213,7 @@ def statx(endpoint, fileref, userid, versioninv=1): rcv, infov = _getxrdfor(endpoint).query(QueryCode.OPAQUEFILE, _getfilepath(verFolder) + ownerarg + '&mgm.pcmd=stat', timeout=timeout) tend = time.time() - infov = infov.decode() + infov = infov.decode() if infov else '' try: if OK_MSG not in str(rcv) or 'retc=2' in infov: # the version folder does not exist: create it (on behalf of the owner) as it is done in Reva @@ -224,7 +224,7 @@ def statx(endpoint, fileref, userid, versioninv=1): rcv, infov = _getxrdfor(endpoint).query(QueryCode.OPAQUEFILE, _getfilepath(verFolder) + ownerarg + '&mgm.pcmd=stat', timeout=timeout) tend = time.time() - infov = infov.decode() + infov = infov.decode() if infov else '' if OK_MSG not in str(rcv) or 'retc=' in infov: raise IOError(rcv) # infov is a full record according to https://gitlab.cern.ch/dss/eos/-/blob/master/mgm/XrdMgmOfs/fsctl/Stat.cc#L53 From fe4f6cc8c21ae8451449249dbec4f0c2b448f94e Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 22 Oct 2022 15:38:57 +0200 Subject: [PATCH 057/325] bridged apps: moved checking/loading of the plugin to the bridge appopen method where it belongs --- src/bridge/__init__.py | 16 +++++++++++++--- src/wopiserver.py | 16 +++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 145df6fd..bdb36624 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -135,13 +135,23 @@ def _gendocid(wopisrc): # The Bridge endpoints start here ############################################################################################################# -def appopen(wopisrc, acctok, appname, viewmode, revatok=None): +def appopen(wopisrc, acctok, appmd, viewmode, revatok=None): '''Open a doc by contacting the provided WOPISrc with the given access_token. Returns a (app-url, params{}) pair if successful, raises a FailedOpen exception otherwise''' wopisrc = urlparse.unquote_plus(wopisrc) if not isinstance(acctok, str): # TODO when using the wopiopen.py tool, the access token has to be decoded, to be clarified acctok = acctok.decode() + + # load plugin if not done at init time and validate URLs + appname, appurl, appinturl, apikey = appmd + try: + WB.loadplugin(appname, appurl, appinturl, apikey) + except ValueError: + raise FailedOpen('Failed to load WOPI bridge plugin for %s' % appname, http.client.INTERNAL_SERVER_ERROR) + except KeyError: + raise FailedOpen('Bridged app %s already configured with a different appurl' % appname, http.client.NOT_IMPLEMENTED) + # WOPI GetFileInfo res = wopic.request(wopisrc, acctok, 'GET') if res.status_code != http.client.OK: @@ -150,10 +160,10 @@ def appopen(wopisrc, acctok, appname, viewmode, revatok=None): filemd = res.json() app = WB.plugins.get(appname.lower()) if not app: - WB.log.warning('msg="Open: appname not supported or missing plugin" filename="%s" appname="%s" token="%s"' % + WB.log.warning('msg="BridgeOpen: appname not supported or missing plugin" filename="%s" appname="%s" token="%s"' % (filemd['BaseFileName'], appname, acctok[-20:])) raise FailedOpen('File type not supported', http.client.BAD_REQUEST) - WB.log.debug('msg="Processing open in supported app" appname="%s" plugin="%s"' % (appname, app)) + WB.log.debug('msg="BridgeOpen: processing supported app" appname="%s" plugin="%s"' % (appname, app)) try: # use the 'UserCanWrite' attribute to decide whether the file is to be opened in read-only mode diff --git a/src/wopiserver.py b/src/wopiserver.py index aa87dc6b..a4cdd43b 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -326,17 +326,6 @@ def iopOpenInApp(): Wopi.log.warning('msg="iopOpenInApp: app-related arguments must be provided" client="%s"' % req.remote_addr) return 'Missing appname or appurl arguments', http.client.BAD_REQUEST - if bridge.issupported(appname): - # This is a bridge-supported application, get the extra info to enable it - apikey = req.headers.get('ApiKey') - appinturl = url_unquote_plus(req.args.get('appinturl', appurl)) # defaults to the external appurl - try: - bridge.WB.loadplugin(appname, appurl, appinturl, apikey) - except ValueError: - return 'Failed to load WOPI bridge plugin for %s' % appname, http.client.INTERNAL_SERVER_ERROR - except KeyError: - return 'Bridged app %s already configured with a different appurl' % appname, http.client.NOT_IMPLEMENTED - try: userid = storage.getuseridfromcreds(usertoken, wopiuser) if userid != usertoken: @@ -354,8 +343,9 @@ def iopOpenInApp(): res = {} if bridge.issupported(appname): try: - res['app-url'], res['form-parameters'] = bridge.appopen(utils.generateWopiSrc(inode), acctok, appname, viewmode, - usertoken) + res['app-url'], res['form-parameters'] = bridge.appopen(utils.generateWopiSrc(inode), acctok, + (appname, appurl, url_unquote_plus(req.args.get('appinturl', appurl), req.headers.get('ApiKey'))), + viewmode, usertoken) except bridge.FailedOpen as foe: return foe.msg, foe.statuscode else: From 2e92a65c71fdb958cdf5f814c5275b8b443f7ebe Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 22 Oct 2022 15:42:48 +0200 Subject: [PATCH 058/325] Bridged apps: removed temporary workaround The system works provided that all plugins must be loaded at init time, otherwise it may happen (e.g. after a restart of the wopiserver) that a legitimate save operation comes before the corresponding plugin had a chance to be initialized (via an open call). --- src/bridge/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index bdb36624..7fe1157f 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -246,9 +246,6 @@ def appsave(docid): except ValueError: WB.log.error('msg="BridgeSave: unknown application" address="%s" appheader="%s" args="%s"' % (flask.request.remote_addr, flask.request.headers.get(BRIDGED_APP_HEADER), flask.request.args)) - # temporary override - appname = 'CodiMD' - #return wopic.jsonify('Unknown application, could not save. %s' % RECOVER_MSG), http.client.UNAUTHORIZED # decide whether to notify the save thread donotify = isclose or wopisrc not in WB.openfiles or WB.openfiles[wopisrc]['lastsave'] < time.time() - WB.saveinterval From de55476252de44bcbb3baa8f70d1e87c682e3a9e Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 22 Oct 2022 15:58:53 +0200 Subject: [PATCH 059/325] CodiMD: fixed retrieval of redirected URLs --- src/bridge/codimd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 25f1cc49..f215eee6 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -153,7 +153,7 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): log.error('msg="Unable to push read-only document to CodiMD" token="%s" response="%d"' % (acctok[-20:], res.status_code)) raise AppFailure - docid = urlparse.urlsplit(res.next.url).path.split('/')[-1] + docid = urlparse.urlsplit(res.headers['location']).path.split('/')[-1] log.info('msg="Pushed read-only document to CodiMD" docid="%s" token="%s"' % (docid, acctok[-20:])) else: # reserve the given docid in CodiMD via a HEAD request @@ -166,7 +166,7 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): raise AppFailure # check if the target docid is real or is a redirect if res.status_code == http.client.FOUND: - newdocid = urlparse.urlsplit(res.next.url).path.split('/')[-1] + newdocid = urlparse.urlsplit(res.headers['location']).path.split('/')[-1] log.info('msg="Document got aliased in CodiMD" olddocid="%s" docid="%s" token="%s"' % (docid, newdocid, acctok[-20:])) docid = newdocid From 3dec61d18390f2573053bd87ddbf1641340c5e54 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 22 Oct 2022 16:40:37 +0200 Subject: [PATCH 060/325] CodiMD: fixed logging --- src/bridge/codimd.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index f215eee6..dca7aec5 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -147,7 +147,7 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): headers={'Content-Type': 'text/markdown'}, verify=sslverify) if res.status_code == http.client.REQUEST_ENTITY_TOO_LARGE: - log.error('msg="File is too large to be edited in CodiMD" token="%s"') + log.error('msg="File is too large to be edited in CodiMD" token="%s"' % acctok[-20:]) raise AppFailure(TOOLARGE) if res.status_code != http.client.FOUND: log.error('msg="Unable to push read-only document to CodiMD" token="%s" response="%d"' % @@ -155,6 +155,7 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): raise AppFailure docid = urlparse.urlsplit(res.headers['location']).path.split('/')[-1] log.info('msg="Pushed read-only document to CodiMD" docid="%s" token="%s"' % (docid, acctok[-20:])) + else: # reserve the given docid in CodiMD via a HEAD request res = requests.head(appurl + '/' + docid, @@ -164,14 +165,14 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): log.error('msg="Unable to reserve note hash in CodiMD" token="%s" response="%d"' % (acctok[-20:], res.status_code)) raise AppFailure + # check if the target docid is real or is a redirect if res.status_code == http.client.FOUND: newdocid = urlparse.urlsplit(res.headers['location']).path.split('/')[-1] log.info('msg="Document got aliased in CodiMD" olddocid="%s" docid="%s" token="%s"' % (docid, newdocid, acctok[-20:])) docid = newdocid - else: - log.debug('msg="Got note hash from CodiMD" docid="%s"' % docid) + # push the document to CodiMD with the update API res = requests.put(appurl + '/api/notes/' + docid, json={'content': mddoc.decode()}, @@ -180,12 +181,13 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): # the file got unlocked because of no activity, yet some user is there: let it go log.warning('msg="Document was being edited in CodiMD, redirecting user" token"%s"' % acctok[-20:]) elif res.status_code == http.client.REQUEST_ENTITY_TOO_LARGE: - log.error('msg="File is too large to be edited in CodiMD" token="%s"') + log.error('msg="File is too large to be edited in CodiMD" docid="%s" token="%s"' % (docid, acctok[-20:])) raise AppFailure(TOOLARGE) elif res.status_code != http.client.OK: - log.error('msg="Unable to push document to CodiMD" token="%s" response="%d"' % - (acctok[-20:], res.status_code)) + log.error('msg="Unable to push document to CodiMD" docid="%s" token="%s" response="%d"' % + (docid, acctok[-20:], res.status_code)) raise AppFailure + log.info('msg="Pushed document to CodiMD" docid="%s" token="%s"' % (docid, acctok[-20:])) except requests.exceptions.ConnectionError as e: log.error('msg="Exception raised attempting to connect to CodiMD" exception="%s"' % e) From be08264bd0ed4e5f5cd1a90bbc11f7d02d79610b Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 22 Oct 2022 17:08:08 +0200 Subject: [PATCH 061/325] Further refined heuristic to identify when conflicted sessions get resolved --- src/core/wopiutils.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 4ea3feea..9101d5c2 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -123,13 +123,20 @@ def validateAndLogHeaders(op): return 'Invalid or expired X-WOPI-Timestamp header', http.client.INTERNAL_SERVER_ERROR # log all relevant headers to help debugging + session = flask.request.headers.get('X-WOPI-SessionId') log.debug('msg="%s: client context" user="%s" filename="%s" token="%s" client="%s" deviceId="%s" reqId="%s" sessionId="%s" ' 'app="%s" appEndpoint="%s" correlationId="%s" wopits="%s"' % (op.title(), acctok['userid'][-20:], acctok['filename'], flask.request.args['access_token'][-20:], flask.request.headers.get('X-Real-Ip', flask.request.remote_addr), flask.request.headers.get('X-WOPI-DeviceId'), flask.request.headers.get('X-Request-Id'), - flask.request.headers.get('X-WOPI-SessionId'), flask.request.headers.get('X-WOPI-RequestingApplication'), + session, flask.request.headers.get('X-WOPI-RequestingApplication'), flask.request.headers.get('X-WOPI-AppEndpoint'), flask.request.headers.get('X-WOPI-CorrelationId'), wopits)) + + # update bookkeeping of pending sessions + if op.title() == 'Checkfileinfo' and session in srv.conflictsessions['pending'] and \ + time.mktime(time.strptime(srv.conflictsessions['pending'][session]['time'])) < time.time() - 900: + # a previously conflicted session is still around executing Checkfileinfo after 15 minutes, assume it got resolved + _resolveSession(session, acctok['filename']) return acctok, None @@ -370,19 +377,25 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, endpoint return resp -def makeLockSuccessResponse(operation, acctok, lock, version): - '''Generates and logs an HTTP 200 response with appropriate headers for Lock/RefreshLock operations''' - session = flask.request.headers.get('X-WOPI-SessionId') +def _resolveSession(session, filename): + '''Mark a session as resolved and account the given filename in the openfiles map. + This is only used for bookkeeping, no functionality is associated to those maps''' if session in srv.conflictsessions['pending']: srv.conflictsessions['pending'].pop(session) srv.conflictsessions['resolved'][session] = time.asctime() # keep some accounting of the open files + if filename not in srv.openfiles: + srv.openfiles[filename] = (time.asctime(), set()) + if session not in srv.openfiles[filename][1]: + srv.openfiles[filename][1].add(session) + + +def makeLockSuccessResponse(operation, acctok, lock, version): + '''Generates and logs an HTTP 200 response with appropriate headers for Lock/RefreshLock operations''' + session = flask.request.headers.get('X-WOPI-SessionId') if not session: session = acctok['username'] - if acctok['filename'] not in srv.openfiles: - srv.openfiles[acctok['filename']] = (time.asctime(), set()) - if session not in srv.openfiles[acctok['filename']][1]: - srv.openfiles[acctok['filename']][1].add(session) + _resolveSession(session, acctok['filename']) log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s"' % (operation.title(), acctok['filename'], flask.request.args['access_token'][-20:], session, lock)) @@ -398,16 +411,9 @@ def storeWopiFile(acctok, retrievedlock, xakey, targetname=''): if not targetname: targetname = acctok['filename'] session = flask.request.headers.get('X-WOPI-SessionId') - if session in srv.conflictsessions['pending']: - srv.conflictsessions['pending'].pop(session) - srv.conflictsessions['resolved'][session] = time.asctime() - # keep some accounting of the open files if not session: session = acctok['username'] - if targetname not in srv.openfiles: - srv.openfiles[targetname] = (time.asctime(), set()) - if session not in srv.openfiles[targetname][1]: - srv.openfiles[targetname][1].add(session) + _resolveSession(session, targetname) st.writefile(acctok['endpoint'], targetname, acctok['userid'], flask.request.get_data(), encodeLock(retrievedlock)) # save the current time for later conflict checking: this is never older than the mtime of the file From 01f472832515cfc31f092bb88d8df497d7688566 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sun, 23 Oct 2022 00:00:12 +0200 Subject: [PATCH 062/325] Also consider Unlock operations as resolution for conflicted sessions --- src/core/wopi.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 0eb499b1..40d18b86 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -310,9 +310,13 @@ def unlock(fileid, reqheaders, acctok): # ignore, it's not worth to report anything here pass - # and update our internal list of opened files + # and update our internal lists of opened files and conflicted sessions try: del srv.openfiles[acctok['filename']] + session = flask.request.headers.get('X-WOPI-SessionId') + if session in srv.conflictsessions['pending']: + srv.conflictsessions['pending'].pop(session) + srv.conflictsessions['resolved'][session] = time.asctime() except KeyError: # already removed? pass From 720cb1ce1ce5ab7777a3e470207ab6109c7ccbb0 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sun, 23 Oct 2022 00:20:52 +0200 Subject: [PATCH 063/325] Fix for commit fe4f6cc --- src/wopiserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wopiserver.py b/src/wopiserver.py index a4cdd43b..0515e600 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -344,7 +344,7 @@ def iopOpenInApp(): if bridge.issupported(appname): try: res['app-url'], res['form-parameters'] = bridge.appopen(utils.generateWopiSrc(inode), acctok, - (appname, appurl, url_unquote_plus(req.args.get('appinturl', appurl), req.headers.get('ApiKey'))), + (appname, appurl, url_unquote_plus(req.args.get('appinturl', appurl)), req.headers.get('ApiKey')), viewmode, usertoken) except bridge.FailedOpen as foe: return foe.msg, foe.statuscode From 8e94d0fc95cb5bdb694bd1326d24a710ac285d22 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sun, 23 Oct 2022 00:35:01 +0200 Subject: [PATCH 064/325] Stricter check for odf files when opened in MS --- src/core/wopiutils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 9101d5c2..b14a2b87 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -212,9 +212,9 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app # preview mode assumes read/write privileges for the acctok viewmode = ViewMode.READ_WRITE if srv.config.get('general', 'disablemswriteodf', fallback='False').upper() == 'TRUE' and \ - fext in srv.codetypes and appname not in ('Collabora', '') and viewmode == ViewMode.READ_WRITE: - # we're opening an ODF file and the app is not Collabora (the last check is needed because the legacy endpoint - # does not set appname when the app is not proxied, so we optimistically assume it's Collabora and let it go) + fext[1:2] in ('od', 'ot') and appname not in ('Collabora', '') and viewmode == ViewMode.READ_WRITE: + # we're opening an ODF (`.o[d|t]?`) file and the app is not Collabora (the appname may be empty because the legacy + # endpoint does not set appname when the app is not proxied, so we optimistically assume it's Collabora and let it go) log.info('msg="Forcing read-only access to ODF file" filename="%s"' % statinfo['filepath']) viewmode = ViewMode.READ_ONLY acctok = jwt.encode({'userid': userid, 'wopiuser': wopiuser, 'filename': statinfo['filepath'], 'fileid': fileid, From f39799c5ddd01b19c392e77245bf853a3e57db5a Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sun, 23 Oct 2022 01:02:17 +0200 Subject: [PATCH 065/325] xroot: better fix on top of 172b77b --- src/core/xrootiface.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index bfe7736d..03cfb2f3 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -213,8 +213,10 @@ def statx(endpoint, fileref, userid, versioninv=1): rcv, infov = _getxrdfor(endpoint).query(QueryCode.OPAQUEFILE, _getfilepath(verFolder) + ownerarg + '&mgm.pcmd=stat', timeout=timeout) tend = time.time() - infov = infov.decode() if infov else '' try: + if not infov: + raise IOError('xrdquery returned nothing, rcv=%s' + rcv) + infov = infov.decode() if OK_MSG not in str(rcv) or 'retc=2' in infov: # the version folder does not exist: create it (on behalf of the owner) as it is done in Reva rcmkdir = _getxrdfor(endpoint).mkdir(_getfilepath(verFolder) + ownerarg, MkDirFlags.MAKEPATH, timeout=timeout) @@ -224,7 +226,9 @@ def statx(endpoint, fileref, userid, versioninv=1): rcv, infov = _getxrdfor(endpoint).query(QueryCode.OPAQUEFILE, _getfilepath(verFolder) + ownerarg + '&mgm.pcmd=stat', timeout=timeout) tend = time.time() - infov = infov.decode() if infov else '' + if not infov: + raise IOError('xrdquery returned nothing, rcv=%s' + rcv) + infov = infov.decode() if OK_MSG not in str(rcv) or 'retc=' in infov: raise IOError(rcv) # infov is a full record according to https://gitlab.cern.ch/dss/eos/-/blob/master/mgm/XrdMgmOfs/fsctl/Stat.cc#L53 @@ -233,7 +237,7 @@ def statx(endpoint, fileref, userid, versioninv=1): (endpoint, _getfilepath(verFolder), str(rcv).strip('\n'), infov, (tend-tstart)*1000)) except IOError as e: # here we should really raise the error, but for now we just log it - log.error('msg="Failed to mkdir/stat version folder, returning file metadata instead" filepath="%s" rc="%s"' % + log.error('msg="Failed to mkdir/stat version folder, returning file metadata instead" filepath="%s" error="%s"' % (_getfilepath(filepath), e)) # return the metadata of the given file, with the inode taken from the version folder endpoint = _geturlfor(endpoint) From 24f3d6ae291f36cf1609ce1a8267df53b5100a24 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sun, 23 Oct 2022 09:50:21 +0200 Subject: [PATCH 066/325] Ensure all elapsedTime logging consistently refers to metadata ops This was not the case for write operations. Logging was also amended for the xroot interface. --- src/core/cs3iface.py | 4 ++-- src/core/localiface.py | 3 ++- src/core/xrootiface.py | 13 ++++++++----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index 89d9a194..a7e4fc33 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -268,6 +268,7 @@ def readfile(endpoint, filepath, userid, lockid): log.error('msg="Failed to initiateFileDownload on read" filepath="%s" trace="%s" code="%s" reason="%s"' % (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) yield IOError(res.status.message) + tend = time.time() log.debug('msg="readfile: InitiateFileDownloadRes returned" trace="%s" protocols="%s"' % (res.status.trace, res.protocols)) @@ -282,7 +283,6 @@ def readfile(endpoint, filepath, userid, lockid): except requests.exceptions.RequestException as e: log.error('msg="Exception when downloading file from Reva" reason="%s"' % e) yield IOError(e) - tend = time.time() data = fileget.content if fileget.status_code != http.client.OK: log.error('msg="Error downloading file from Reva" code="%d" reason="%s"' % @@ -315,6 +315,7 @@ def writefile(endpoint, filepath, userid, content, lockid, islock=False): log.error('msg="Failed to initiateFileUpload on write" filepath="%s" trace="%s" code="%s" reason="%s"' % (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) + tend = time.time() log.debug('msg="writefile: InitiateFileUploadRes returned" trace="%s" protocols="%s"' % (res.status.trace, res.protocols)) @@ -330,7 +331,6 @@ def writefile(endpoint, filepath, userid, content, lockid, islock=False): except requests.exceptions.RequestException as e: log.error('msg="Exception when uploading file to Reva" reason="%s"' % e) raise IOError(e) - tend = time.time() if putres.status_code == http.client.UNAUTHORIZED: log.warning('msg="Access denied uploading file to Reva" reason="%s"' % putres.reason) raise IOError(common.ACCESS_ERROR) diff --git a/src/core/localiface.py b/src/core/localiface.py index 771a8c69..9eab879f 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -231,6 +231,7 @@ def writefile(endpoint, filepath, userid, content, lockid, islock=False): # so we resort to the os-level open(), with some caveats fd = os.open(_getfilepath(filepath), os.O_CREAT | os.O_EXCL) f = os.fdopen(fd, mode='wb') + tend = time.time() written = f.write(content) # os.write(fd, ...) raises EBADF? os.close(fd) # f.close() raises EBADF! while this works # as f goes out of scope here, we'd get a false ResourceWarning, which is ignored by the above filter @@ -243,11 +244,11 @@ def writefile(endpoint, filepath, userid, content, lockid, islock=False): else: try: with open(_getfilepath(filepath), mode='wb') as f: + tend = time.time() written = f.write(content) except OSError as e: log.error('msg="Error writing file" filepath="%s" error="%s"' % (filepath, e)) raise IOError(e) - tend = time.time() if written != size: raise IOError('Written %d bytes but content is %d bytes' % (written, size)) log.info('msg="File written successfully" filepath="%s" elapsedTimems="%.1f" islock="%s"' % diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 03cfb2f3..29a1d345 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -374,20 +374,23 @@ def writefile(endpoint, filepath, userid, content, _lockid, islock=False): log.info('msg="File exists on write but islock flag requested" filepath="%s"' % filepath) raise IOError(common.EXCL_ERROR) # any other failure is reported as is - log.warning('msg="Error opening the file for write" filepath="%s" error="%s"' % (filepath, rc.message.strip('\n'))) + log.error('msg="Error opening the file for write" filepath="%s" elapsedTimems="%.1f" error="%s"' % + (filepath, (tend-tstart)*1000, rc.message.strip('\n'))) raise IOError(rc.message.strip('\n')) - # write the file. In a future implementation, we should find a way to only update the required chunks... rc, _ = f.write(content, offset=0, size=size) if not rc.ok: - log.warning('msg="Error writing the file" filepath="%s" error="%s"' % (filepath, rc.message.strip('\n'))) + log.error('msg="Error writing the file" filepath="%s" elapsedTimems="%.1f" error="%s"' % + (filepath, (tend-tstart)*1000, rc.message.strip('\n'))) raise IOError(rc.message.strip('\n')) rc, _ = f.truncate(size) if not rc.ok: - log.warning('msg="Error truncating the file" filepath="%s" error="%s"' % (filepath, rc.message.strip('\n'))) + log.error('msg="Error truncating the file" filepath="%s" elapsedTimems="%.1f" error="%s"' % + (filepath, (tend-tstart)*1000, rc.message.strip('\n'))) raise IOError(rc.message.strip('\n')) rc, _ = f.close() if not rc.ok: - log.warning('msg="Error closing the file" filepath="%s" error="%s"' % (filepath, rc.message.strip('\n'))) + log.error('msg="Error closing the file" filepath="%s" elapsedTimems="%.1f" error="%s"' % + (filepath, (tend-tstart)*1000, rc.message.strip('\n'))) raise IOError(rc.message.strip('\n')) if existingLock: try: From 83fc7e9d2aa12bbd46b3c2b55bf32be74be37b6f Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 24 Oct 2022 10:39:16 +0200 Subject: [PATCH 067/325] Better specification of the usage of the downloadurl config entry --- wopiserver.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wopiserver.conf b/wopiserver.conf index 85e93eab..90165800 100644 --- a/wopiserver.conf +++ b/wopiserver.conf @@ -39,8 +39,10 @@ loghandler = file #brandingurl = # URL for direct download of files. The complete URL that is sent -# to clients will include the access_token argument -#downloadurl = https://your-wopi-server.org/wopi/cbox/download +# to WOPI apps will include the access_token argument. +# A route to the /wopi/iop/download endpoint could be used for this, +# though WOPI apps also work without this route configured. +#downloadurl = # Optional URL to display a file sharing dialog. This enables # a 'Share' button within the application. The URL may contain From 296c7dea43eb2445625790e2224a07ffc5dc3f47 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 24 Oct 2022 18:09:11 +0200 Subject: [PATCH 068/325] More aggressive resolution of pending conflicted sessions --- src/core/wopiutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index b14a2b87..7bff56da 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -134,8 +134,8 @@ def validateAndLogHeaders(op): # update bookkeeping of pending sessions if op.title() == 'Checkfileinfo' and session in srv.conflictsessions['pending'] and \ - time.mktime(time.strptime(srv.conflictsessions['pending'][session]['time'])) < time.time() - 900: - # a previously conflicted session is still around executing Checkfileinfo after 15 minutes, assume it got resolved + time.mktime(time.strptime(srv.conflictsessions['pending'][session]['time'])) < time.time() - 300: + # a previously conflicted session is still around executing Checkfileinfo after 5 minutes, assume it got resolved _resolveSession(session, acctok['filename']) return acctok, None From 9d739445be6a2851d321fd21c9f0f078ff58908e Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 24 Oct 2022 18:29:44 +0200 Subject: [PATCH 069/325] Log real remote IPs more consistently + ensure missing access tokens are logged correctly --- src/core/wopiutils.py | 11 ++++++++--- src/wopiserver.py | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 7bff56da..ecff6678 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -27,6 +27,9 @@ # this is the xattr key used for conflicts resolution on the remote storage LASTSAVETIMEKEY = 'iop.wopi.lastwritetime' +# header used by reverse proxies such as traefik to pass the real remote IP address +REALIPHEADER = 'X-Real-IP' + # convenience references to global entities st = None srv = None @@ -90,7 +93,8 @@ def logGeneralExceptionAndReturn(ex, req): '''Convenience function to log a stack trace and return HTTP 500''' ex_type, ex_value, ex_traceback = sys.exc_info() log.critical('msg="Unexpected exception caught" exception="%s" type="%s" traceback="%s" client="%s" requestedUrl="%s"' % - (ex, ex_type, traceback.format_exception(ex_type, ex_value, ex_traceback), req.remote_addr, req.url)) + (ex, ex_type, traceback.format_exception(ex_type, ex_value, ex_traceback), + flask.request.headers.get(REALIPHEADER, flask.request.remote_addr), req.url)) return 'Internal error, please contact support', http.client.INTERNAL_SERVER_ERROR @@ -104,7 +108,8 @@ def validateAndLogHeaders(op): raise jwt.exceptions.ExpiredSignatureError except (jwt.exceptions.DecodeError, jwt.exceptions.ExpiredSignatureError, KeyError) as e: log.info('msg="Expired or malformed token" client="%s" requestedUrl="%s" error="%s" token="%s"' % - (flask.request.remote_addr, flask.request.base_url, str(type(e)) + ': ' + str(e), flask.request.args['access_token'])) + (flask.request.headers.get(REALIPHEADER, flask.request.remote_addr), flask.request.base_url, + str(type(e)) + ': ' + str(e), flask.request.args.get('access_token'))) return 'Invalid access token', http.client.UNAUTHORIZED # validate the WOPI timestamp: this is typically not present, but if it is we must check its expiration @@ -127,7 +132,7 @@ def validateAndLogHeaders(op): log.debug('msg="%s: client context" user="%s" filename="%s" token="%s" client="%s" deviceId="%s" reqId="%s" sessionId="%s" ' 'app="%s" appEndpoint="%s" correlationId="%s" wopits="%s"' % (op.title(), acctok['userid'][-20:], acctok['filename'], - flask.request.args['access_token'][-20:], flask.request.headers.get('X-Real-Ip', flask.request.remote_addr), + flask.request.args['access_token'][-20:], flask.request.headers.get(REALIPHEADER, flask.request.remote_addr), flask.request.headers.get('X-WOPI-DeviceId'), flask.request.headers.get('X-Request-Id'), session, flask.request.headers.get('X-WOPI-RequestingApplication'), flask.request.headers.get('X-WOPI-AppEndpoint'), flask.request.headers.get('X-WOPI-CorrelationId'), wopits)) diff --git a/src/wopiserver.py b/src/wopiserver.py index 0515e600..956c52b5 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -465,7 +465,8 @@ def wopiFilesPost(fileid): op = headers['X-WOPI-Override'] # must be one of the following strings, throws KeyError if missing except KeyError as e: Wopi.log.warning('msg="Missing argument" client="%s" requestedUrl="%s" error="%s" token="%s"' % - (flask.request.remote_addr, flask.request.base_url, e, flask.request.args.get('access_token')[-20:])) + (flask.request.headers.get(utils.REALIPHEADER, flask.request.remote_addr), flask.request.base_url, + e, flask.request.args.get('access_token'))) return 'Missing argument', http.client.BAD_REQUEST acctokOrMsg, httpcode = utils.validateAndLogHeaders(op) if httpcode: From da9d337b4db5f934e32eeb2d4aadb151206febef Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 24 Oct 2022 18:36:39 +0200 Subject: [PATCH 070/325] Fixed check for ODF, cf. 8e94d0f --- src/core/wopiutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index ecff6678..cb8c760a 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -217,7 +217,7 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app # preview mode assumes read/write privileges for the acctok viewmode = ViewMode.READ_WRITE if srv.config.get('general', 'disablemswriteodf', fallback='False').upper() == 'TRUE' and \ - fext[1:2] in ('od', 'ot') and appname not in ('Collabora', '') and viewmode == ViewMode.READ_WRITE: + fext[1:3] in ('od', 'ot') and appname not in ('Collabora', '') and viewmode == ViewMode.READ_WRITE: # we're opening an ODF (`.o[d|t]?`) file and the app is not Collabora (the appname may be empty because the legacy # endpoint does not set appname when the app is not proxied, so we optimistically assume it's Collabora and let it go) log.info('msg="Forcing read-only access to ODF file" filename="%s"' % statinfo['filepath']) From 17a22637744054062fe54c2c4065790bc0d9c1af Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 26 Oct 2022 09:58:19 +0200 Subject: [PATCH 071/325] Bridged apps: reattempt to save only on transient errors + restored error message --- src/bridge/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 7fe1157f..0fee6be8 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -246,6 +246,7 @@ def appsave(docid): except ValueError: WB.log.error('msg="BridgeSave: unknown application" address="%s" appheader="%s" args="%s"' % (flask.request.remote_addr, flask.request.headers.get(BRIDGED_APP_HEADER), flask.request.args)) + return wopic.jsonify('Unknown application, could not save. %s' % RECOVER_MSG), http.client.BAD_REQUEST # decide whether to notify the save thread donotify = isclose or wopisrc not in WB.openfiles or WB.openfiles[wopisrc]['lastsave'] < time.time() - WB.saveinterval @@ -364,7 +365,7 @@ def savedirty(self, openfile, wopisrc): WB.saveresponses[wopisrc] = WB.plugins[appname].savetostorage(wopisrc, openfile['acctok'], _intersection(openfile['toclose']), wopilock) openfile['lastsave'] = int(time.time()) - if WB.saveresponses[wopisrc][1] >= http.client.BAD_REQUEST: + if WB.saveresponses[wopisrc][1] == http.client.FAILED_DEPENDENCY: # this is hopefully transient, yet we need to try until we get the file back to storage: # the updated lastsave time ensures next retry will happen after the saveinterval time if 'still-dirty' not in openfile['toclose']: From 02adb60ec85f7060dd953cfbc384ec1ed21455f6 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 27 Oct 2022 10:46:43 +0200 Subject: [PATCH 072/325] Bridged apps: better error coverage in case of transient errors --- src/bridge/wopiclient.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/bridge/wopiclient.py b/src/bridge/wopiclient.py index 1f8e0e3f..084e9ffb 100644 --- a/src/bridge/wopiclient.py +++ b/src/bridge/wopiclient.py @@ -192,12 +192,17 @@ def relock(wopisrc, acctok, docid, isclose): def handleputfile(wopicall, wopisrc, res): '''Deal with conflicts or errors following a PutFile/PutRelative request''' if res.status_code == http.client.CONFLICT: + # this is typically a user issue, return 500 and stop further editing log.warning('msg="Conflict when calling WOPI %s" url="%s" reason="%s"' % (wopicall, wopisrc, res.headers.get('X-WOPI-LockFailureReason'))) return jsonify('Error saving the file. %s' % res.headers.get('X-WOPI-LockFailureReason')), http.client.INTERNAL_SERVER_ERROR + if res.status_code == http.client.INTERNAL_SERVER_ERROR: + # hopefully this is transient and the server has kept a local copy for later recovery + log.error('msg="Calling WOPI %s failed, will retry" url="%s" response="%s"' % (wopicall, wopisrc, res.status_code)) + return jsonify('Error saving the file, will try again'), http.client.FAILED_DEPENDENCY if res.status_code != http.client.OK: - # hopefully the server has kept a local copy for later recovery + # any other error is considered also fatal log.error('msg="Calling WOPI %s failed" url="%s" response="%s"' % (wopicall, wopisrc, res.status_code)) return jsonify('Error saving the file, please contact support'), http.client.INTERNAL_SERVER_ERROR return None From fb2807022a0827d244f702c3cd026902f80e8802 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 27 Oct 2022 10:57:35 +0200 Subject: [PATCH 073/325] Made PutFile more robust to transient write errors It has been seen that if a write times out yet "touches" the file, then the lastwritetime xattr is not updated, and further save operations are blocked. This makes sure to attempt to update the xattr. Of course if the latter fails too, all bets are off. --- src/core/wopiutils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index cb8c760a..f48c6956 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -420,9 +420,16 @@ def storeWopiFile(acctok, retrievedlock, xakey, targetname=''): session = acctok['username'] _resolveSession(session, targetname) - st.writefile(acctok['endpoint'], targetname, acctok['userid'], flask.request.get_data(), encodeLock(retrievedlock)) + writeerror = None + try: + st.writefile(acctok['endpoint'], targetname, acctok['userid'], flask.request.get_data(), encodeLock(retrievedlock)) + except IOError as e: + # in case something goes wrong on write, we still want to setxattr but report this error to the caller + writeerror = e # save the current time for later conflict checking: this is never older than the mtime of the file st.setxattr(acctok['endpoint'], targetname, acctok['userid'], xakey, int(time.time()), encodeLock(retrievedlock)) + if writeerror: + raise writeerror def storeAfterConflict(acctok, retrievedlock, lock, reason): From 77f6658cea0a54ffd19f0ab10d7c5eddb5fb55f9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 25 Oct 2022 18:23:23 +0200 Subject: [PATCH 074/325] Disable SaveAs when user is not owner --- src/core/wopi.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 40d18b86..f92d788e 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -62,6 +62,7 @@ def checkFileInfo(fileid, acctok): # this is the top level public share, which is anonymous fmd['BreadcrumbFolderName'] = 'Public share' else: + fmd['IsAnonymousUser'] = False fmd['UserFriendlyName'] = acctok['username'] fmd['BreadcrumbFolderName'] = 'Back to ' + os.path.dirname(acctok['filename']) if furl == '/': # if no target folder URL was given, override the above and completely hide it @@ -80,7 +81,12 @@ def checkFileInfo(fileid, acctok): fmd['SupportsExtendedLockLength'] = fmd['SupportsGetLock'] = True fmd['SupportsUpdate'] = fmd['UserCanWrite'] = fmd['SupportsLocks'] = \ fmd['SupportsDeleteFile'] = acctok['viewmode'] == utils.ViewMode.READ_WRITE - fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE + # SaveAs functionality is enabled only for owners and in READ_WRITE mode. Anonymous and federated users + # are never owners despite their wopiuser credentials may match the owner's. Federated/external accounts + # are given by Reva with their network domain in parenthesis, that's why we match them. + notOwner = fmd['OwnerId'] != fmd['UserId'] or fmd['IsAnonymousUser'] or \ + ('(' in acctok['username'] and acctok['username'][-1] == ')') + fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE or notOwner fmd['SupportsRename'] = fmd['UserCanRename'] = enablerename and (acctok['viewmode'] == utils.ViewMode.READ_WRITE) fmd['SupportsContainers'] = False # TODO this is all to be implemented fmd['SupportsUserInfo'] = False # TODO https://docs.microsoft.com/en-us/openspecs/office_protocols/ms-wopi/371e25ae-e45b-47ab-aec3-9111e962919d From 56f7fc4ce2eee3d645331061461b8dedf4fb0cc1 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 28 Oct 2022 16:16:45 +0200 Subject: [PATCH 075/325] Sanitize username in storeForRecovery --- src/core/wopiutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index f48c6956..172ecfad 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -471,7 +471,7 @@ def storeAfterConflict(acctok, retrievedlock, lock, reason): def storeForRecovery(content, username, filename, acctokforlog, exception): try: - filepath = srv.recoverypath + os.sep + time.strftime('%Y%m%dT%H%M%S') + '_editedby_' + username \ + filepath = srv.recoverypath + os.sep + time.strftime('%Y%m%dT%H%M%S') + '_editedby_' + secure_filename(username) \ + '_origat_' + secure_filename(filename) with open(filepath, mode='wb') as f: written = f.write(content) From 0765197c36d9e471e60db7691a94b4fa79b876c8 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 2 Nov 2022 09:03:58 +0100 Subject: [PATCH 076/325] Bridged apps: use error-level logging for HTTP 500 responses --- src/bridge/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 0fee6be8..9579d1c0 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -275,7 +275,11 @@ def appsave(docid): # return latest known state for this document if wopisrc in WB.saveresponses: resp = WB.saveresponses[wopisrc] - WB.log.info('msg="BridgeSave: returned response" response="%s" token="%s"' % (resp, acctok[-20:])) + if resp[1] == http.client.INTERNAL_SERVER_ERROR: + logf = WB.log.error + else: + logf = WB.log.info + logf('msg="BridgeSave: returned response" response="%s" token="%s"' % (resp, acctok[-20:])) del WB.saveresponses[wopisrc] return resp WB.log.info('msg="BridgeSave: enqueued action" immediate="%s" token="%s"' % (donotify, acctok[-20:])) From 9377240c494616a124f4ca3555bd315578c8bd79 Mon Sep 17 00:00:00 2001 From: Snyk bot Date: Mon, 7 Nov 2022 10:13:17 +0200 Subject: [PATCH 077/325] [Snyk] Security upgrade wheel from 0.30.0 to 0.38.0 (#99) * fix: requirements.txt to reduce vulnerabilities The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-WHEEL-3092128 Co-authored-by: Giuseppe Lo Presti --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7711cfda..0ef8ce47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ more_itertools prometheus-flask-exporter cs3apis>=0.1.dev101 waitress +wheel>=0.38.0 # not directly required, pinned by Snyk to avoid a vulnerability From 3f17478f992859ffc35e6aecc1c5d9c305550cc8 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 8 Nov 2022 17:54:58 +0100 Subject: [PATCH 078/325] cs3iface: removed legacy config option --- src/core/cs3iface.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index a7e4fc33..57360a6c 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -34,11 +34,7 @@ def init(inconfig, inlog): ctx['ssl_verify'] = inconfig.getboolean('cs3', 'sslverify', fallback=True) ctx['authtokenvalidity'] = inconfig.getint('cs3', 'authtokenvalidity') ctx['lockexpiration'] = inconfig.getint('general', 'wopilockexpiration') - if inconfig.has_option('cs3', 'revagateway'): - revagateway = inconfig.get('cs3', 'revagateway') - else: - # legacy entry, to be dropped at next major release - revagateway = inconfig.get('cs3', 'revahost') + revagateway = inconfig.get('cs3', 'revagateway') # prepare the gRPC connection ch = grpc.insecure_channel(revagateway) ctx['cs3gw'] = cs3gw_grpc.GatewayAPIStub(ch) From a868a3d7b0cd3b74e070b2da72cd084fbeca784e Mon Sep 17 00:00:00 2001 From: "lgtm-com[bot]" <43144390+lgtm-com[bot]@users.noreply.github.com> Date: Fri, 11 Nov 2022 10:39:15 +0100 Subject: [PATCH 079/325] Add CodeQL workflow for GitHub code scanning (#100) Co-authored-by: LGTM Migrator --- .github/workflows/codeql.yml | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..c4bcb6e1 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,41 @@ +name: "CodeQL" + +on: + push: + branches: [ "master", "aarnet", "cernbox", "collaboratest", "eoslock", "nginx", "oolock-autoexpire", "ooracefix", "open-for-iop", "perma-inode", "python3", "snyk-fix-0d459d1e2b96d3ef1c5a96800328793e", "snyk-fix-af3411a9b6ac485b96ed2d8f434e9d97", "syslog", "tus-uploads", "wopi4", "xroot-cs9" ] + pull_request: + branches: [ "master" ] + schedule: + - cron: "44 11 * * 0" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ python ] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + queries: +security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{ matrix.language }}" From e204c9c4c61c44f3fa879ab463413af367ced789 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 11 Nov 2022 09:48:57 +0100 Subject: [PATCH 080/325] Bridged apps: do not attempt to save after a token had expired --- src/bridge/__init__.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 9579d1c0..b168c83d 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -340,26 +340,36 @@ def savedirty(self, openfile, wopisrc): appname = openfile['app'].lower() try: wopilock = wopic.getlock(wopisrc, openfile['acctok']) - except wopic.InvalidLock: + except wopic.InvalidLock as ile1: + if str(ile1) == str(http.client.UNAUTHORIZED): + # this token has expired, nothing we can do any longer: by experience this happens on left-over + # browser sessions, and the file was fully saved. Therefore just clean up by using some 'fake' metadata + WB.log.warning('msg="SaveThread: discarding file as token has expired" token="%s" docid="%s"' % + (openfile['acctok'][-20:], openfile['docid'])) + openfile['lastsave'] = int(time.time()) + openfile['tosave'] = False + openfile['toclose'] = {'invalid-lock': True} + return None + WB.log.info('msg="SaveThread: attempting to relock file" token="%s" docid="%s"' % (openfile['acctok'][-20:], openfile['docid'])) try: wopilock = WB.saveresponses[wopisrc] = wopic.relock( wopisrc, openfile['acctok'], openfile['docid'], _intersection(openfile['toclose'])) - except wopic.InvalidLock as ile: + except wopic.InvalidLock as ile2: # even this attempt failed, give up - WB.saveresponses[wopisrc] = wopic.jsonify(str(ile)), http.client.INTERNAL_SERVER_ERROR + WB.saveresponses[wopisrc] = wopic.jsonify(str(ile2)), http.client.INTERNAL_SERVER_ERROR # attempt to save to local storage to help for later recovery: this is a feature of the core wopiserver content, rc = WB.plugins[appname].savetostorage(wopisrc, openfile['acctok'], False, {'doc': openfile['docid']}, onlyfetch=True) if rc == http.client.OK: utils.storeForRecovery(content, 'unknown', wopisrc[wopisrc.rfind('/') + 1:], - openfile['acctok'][-20:], ile) + openfile['acctok'][-20:], ile2) else: WB.log.error('msg="SaveThread: failed to fetch file for recovery to local storage" ' + 'token="%s" docid="%s" app="%s" response="%s"' % (openfile['acctok'][-20:], openfile['docid'], appname, rc)) - # set some 'fake' metadata, will be automatically cleaned up later + # as above set some 'fake' metadata, will be automatically cleaned up later openfile['lastsave'] = int(time.time()) openfile['tosave'] = False openfile['toclose'] = {'invalid-lock': True} From de069b8d0ada2b8a8d044d43276f2825c1f2967d Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 11 Nov 2022 09:49:26 +0100 Subject: [PATCH 081/325] Improved logging --- src/bridge/wopiclient.py | 2 +- src/core/wopi.py | 7 ++----- src/core/wopiutils.py | 7 ++++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/bridge/wopiclient.py b/src/bridge/wopiclient.py index 084e9ffb..2473834b 100644 --- a/src/bridge/wopiclient.py +++ b/src/bridge/wopiclient.py @@ -89,7 +89,7 @@ def getlock(wopisrc, acctok): # the lock is expected to be a JSON dict, see generatelock() return json.loads(res.headers['X-WOPI-Lock']) except (ValueError, KeyError, json.decoder.JSONDecodeError) as e: - log.warning('msg="Missing or malformed WOPI lock" exception="%s" error="%s"' % (type(e), e)) + log.warning('msg="Missing or malformed WOPI lock" exception="%s: %s"' % (type(e), e)) raise InvalidLock(e) diff --git a/src/core/wopi.py b/src/core/wopi.py index f92d788e..0e7a5b79 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -112,12 +112,9 @@ def checkFileInfo(fileid, acctok): (flask.request.args['access_token'][-20:], fmd)) return res except IOError as e: - log.info('msg="Requested file not found" filename="%s" token="%s" error="%s"' % + log.info('msg="Requested file not found" filename="%s" token="%s" details="%s"' % (acctok['filename'], flask.request.args['access_token'][-20:], e)) return 'File not found', http.client.NOT_FOUND - except KeyError as e: - log.warning('msg="Invalid access token or request argument" error="%s" request="%s"' % (e, flask.request.__dict__)) - return 'Invalid request', http.client.UNAUTHORIZED def getFile(_fileid, acctok): @@ -446,7 +443,7 @@ def deleteFile(fileid, _reqheaders_unused, acctok): except IOError as e: if common.ENOENT_MSG in str(e): return 'File not found', http.client.NOT_FOUND - log.info('msg="DeleteFile" token="%s" error="%s"' % (flask.request.args['access_token'][-20:], e)) + log.error('msg="DeleteFile" token="%s" error="%s"' % (flask.request.args['access_token'][-20:], e)) return IO_ERROR, http.client.INTERNAL_SERVER_ERROR diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 172ecfad..cf401436 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -107,7 +107,7 @@ def validateAndLogHeaders(op): if acctok['exp'] < time.time() or 'cs3org:wopiserver' not in acctok['iss']: raise jwt.exceptions.ExpiredSignatureError except (jwt.exceptions.DecodeError, jwt.exceptions.ExpiredSignatureError, KeyError) as e: - log.info('msg="Expired or malformed token" client="%s" requestedUrl="%s" error="%s" token="%s"' % + log.info('msg="Expired or malformed token" client="%s" requestedUrl="%s" details="%s" token="%s"' % (flask.request.headers.get(REALIPHEADER, flask.request.remote_addr), flask.request.base_url, str(type(e)) + ': ' + str(e), flask.request.args.get('access_token'))) return 'Invalid access token', http.client.UNAUTHORIZED @@ -122,8 +122,9 @@ def validateAndLogHeaders(op): # timestamps older than 20 minutes must be considered expired raise ValueError except ValueError: - log.warning('msg="%s: invalid X-WOPI-Timestamp" user="%s" filename="%s" request="%s"' % - (op, acctok['userid'][-20:], acctok['filename'], flask.request.__dict__)) + log.warning('msg="%s: invalid X-WOPI-Timestamp" user="%s" token="%s" client="%s"' % + (op, acctok['userid'][-20:], flask.request.args['access_token'][-20:], + flask.request.headers.get(REALIPHEADER, flask.request.remote_addr))) # UNAUTHORIZED would seem more appropriate here, but the ProofKeys part of the MS test suite explicitly requires this return 'Invalid or expired X-WOPI-Timestamp header', http.client.INTERNAL_SERVER_ERROR From 75649e024089ae064a6ba3aa7778f9e348a99ef6 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 11 Nov 2022 13:57:16 +0100 Subject: [PATCH 082/325] Fixed missing return as found by CodeQL --- src/core/wopiutils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index cf401436..0b6b8e17 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -349,6 +349,8 @@ def compareWopiLocks(lock1, lock2): log.debug('msg="compareLocks" lock1="%s" lock2="%s" strict="False" result="%r"' % (lock1, lock2, l1['S'] == lock2)) return l1['S'] == lock2 # also used by Word (BUG!) + log.debug('msg="compareLocks" lock1="%s" lock2="%s" strict="False" result="False"' % (lock1, lock2)) + return False except (TypeError, ValueError): # lock1 is not a JSON dictionary: log the lock values and fail the comparison log.debug('msg="compareLocks" lock1="%s" lock2="%s" strict="False" result="False"' % (lock1, lock2)) From 3f841026025b5bd4bc9f866a2b1996a87e82eddd Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 11 Nov 2022 13:57:50 +0100 Subject: [PATCH 083/325] Test: improved assert --- test/test_storageiface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_storageiface.py b/test/test_storageiface.py index 7bb62b34..b8ee979c 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -103,7 +103,7 @@ def test_statx_fileid(self): # detected CS3 storage, test if fileid-based stat is supported # (notably, homepath is not part of the fileid) statInfoId = self.storage.stat(self.endpoint, 'fileid-' + self.username + '%2Ftest.txt', self.userid) - self.assertTrue(statInfo['inode'] == statInfoId['inode']) + self.assertEqual(statInfo['inode'], statInfoId['inode']) self.storage.removefile(self.endpoint, self.homepath + '/test.txt', self.userid) def test_statx_invariant_fileid(self): From 9f6ff5f681f0927bbcc3d104f7305f096ad20130 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 11 Nov 2022 14:11:02 +0100 Subject: [PATCH 084/325] Added comments to address CodeQL issues --- src/bridge/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index b168c83d..e33a3df3 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -210,6 +210,7 @@ def appopen(wopisrc, acctok, appmd, viewmode, revatok=None): try: del WB.saveresponses[wopisrc] except KeyError: + # nothing found, that's fine pass else: # user has no write privileges, just fetch the document and push it to the app on a random docid @@ -268,6 +269,7 @@ def appsave(docid): try: del WB.saveresponses[wopisrc] except KeyError: + # nothing found, that's fine pass if donotify: # note that the save thread stays locked until we release the context, after return! @@ -424,11 +426,12 @@ def cleanup(self, openfile, wopisrc, wopilock): except wopic.InvalidLock: # nothing to do here, this document may have been closed by another wopibridge if openfile['lastsave'] < time.time() - WB.unlockinterval: - # yet cleanup only after the unlockinterval time, cf. the InvalidLock handling in savedirty() + # yet clean up only after the unlockinterval time, cf. the InvalidLock handling in savedirty() WB.log.info('msg="SaveThread: cleaning up metadata, file already unlocked" url="%s"' % wopisrc) try: del WB.openfiles[wopisrc] except KeyError: + # ignore potential races on this item pass return From ed9be99d559e01eb33967ae904ff69979f1450f8 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 11 Nov 2022 14:14:17 +0100 Subject: [PATCH 085/325] Minor improvement in discovery logic following CodeQL scan --- src/core/discovery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/discovery.py b/src/core/discovery.py index db583120..a8589ccb 100644 --- a/src/core/discovery.py +++ b/src/core/discovery.py @@ -91,7 +91,8 @@ def registerapp(appname, appurl, appinturl, apikey=None): # bridge plugin could not be initialized pass except requests.exceptions.ConnectionError: - pass + log.error('msg="Failed to connect to app" appurl="%s"' % appurl) + return # in all other cases, log failure log.error('msg="Attempted to register a non WOPI-compatible app" appurl="%s"' % appurl) From 21f3ba942b024461b3865bf8aabbe6ef72b045aa Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 11 Nov 2022 14:18:02 +0100 Subject: [PATCH 086/325] Improved comments following CodeQL scan --- src/core/wopi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 0e7a5b79..914dfff1 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -382,8 +382,8 @@ def putRelative(fileid, reqheaders, acctok): 'Url': utils.generateWopiSrc(statInfo['inode'], acctok['appname'] == srv.proxiedappname), }) except IOError: + # optimistically assume we're clear pass - # else we can use the relative target targetName = relTarget # either way, we now have a targetName to save the file: attempt to do so try: From 42617cd0b95ba7656f1c7b65ba71d15bb11055f9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 18 Oct 2022 17:15:38 +0200 Subject: [PATCH 087/325] Extended writefile API to include app and lockid as lock metadata --- src/core/cs3iface.py | 8 +++++++- src/core/localiface.py | 10 +++++++--- src/core/wopiutils.py | 5 +++-- src/core/xrootiface.py | 18 +++++++++++++++--- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index 57360a6c..649ec15c 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -290,7 +290,7 @@ def readfile(endpoint, filepath, userid, lockid): yield data[i:i + ctx['chunksize']] -def writefile(endpoint, filepath, userid, content, lockid, islock=False): +def writefile(endpoint, filepath, userid, content, lockmd, islock=False): '''Write a file using the given userid as access token. The entire content is written and any pre-existing file is deleted (or moved to the previous version if supported). The islock flag is currently not supported. The backend should at least support @@ -300,6 +300,10 @@ def writefile(endpoint, filepath, userid, content, lockid, islock=False): tstart = time.time() # prepare endpoint + if lockmd: + _, lockid = lockmd # TODO we are not validating the holder on write, only the lock_id + else: + lockid = None if isinstance(content, str): content = bytes(content, 'UTF-8') size = str(len(content)) @@ -310,6 +314,8 @@ def writefile(endpoint, filepath, userid, content, lockid, islock=False): if res.status.code != cs3code.CODE_OK: log.error('msg="Failed to initiateFileUpload on write" filepath="%s" trace="%s" code="%s" reason="%s"' % (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) + if 'lock' in res.status.message: # TODO find the error code returned by Reva once this is implemented + raise IOError(common.LOCK_MISMATCH_ERROR) raise IOError(res.status.message) tend = time.time() log.debug('msg="writefile: InitiateFileUploadRes returned" trace="%s" protocols="%s"' % diff --git a/src/core/localiface.py b/src/core/localiface.py index 9eab879f..c3530f0e 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -111,9 +111,9 @@ def _checklock(op, endpoint, filepath, userid, lockid): # this is a special value to skip the check, used by the lock operations themselves return lock = getlock(endpoint, filepath, userid) - if lock and lock['lock_id'] != lockid: + if lock and lock['lock_id'] != lockid: # here we could also validate the app/holder log.warning('msg="%s: file was locked" filepath="%s" holder="%s"' % (op, filepath, lock['app_name'])) - raise IOError('File was locked') + raise IOError(common.LOCK_MISMATCH_ERROR) def setxattr(endpoint, filepath, userid, key, value, lockid): @@ -213,13 +213,17 @@ def readfile(_endpoint, filepath, _userid, _lockid): yield IOError(e) -def writefile(endpoint, filepath, userid, content, lockid, islock=False): +def writefile(endpoint, filepath, userid, content, lockmd, islock=False): '''Write a file via xroot on behalf of the given userid. The entire content is written and any pre-existing file is deleted (or moved to the previous version if supported). With islock=True, the file is opened with O_CREAT|O_EXCL.''' if isinstance(content, str): content = bytes(content, 'UTF-8') size = len(content) + if lockmd: + _, lockid = lockmd + else: + lockid = None _checklock('writefile', endpoint, filepath, userid, lockid) log.debug('msg="Invoking writeFile" filepath="%s" size="%d"' % (filepath, size)) tstart = time.time() diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 0b6b8e17..69209fdb 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -425,11 +425,12 @@ def storeWopiFile(acctok, retrievedlock, xakey, targetname=''): writeerror = None try: - st.writefile(acctok['endpoint'], targetname, acctok['userid'], flask.request.get_data(), encodeLock(retrievedlock)) + st.writefile(acctok['endpoint'], targetname, acctok['userid'], flask.request.get_data(), + (acctok['appname'], encodeLock(retrievedlock))) except IOError as e: # in case something goes wrong on write, we still want to setxattr but report this error to the caller writeerror = e - # save the current time for later conflict checking: this is never older than the mtime of the file + # in all cases save the current time for later conflict checking: this is never older than the mtime of the file st.setxattr(acctok['endpoint'], targetname, acctok['userid'], xakey, int(time.time()), encodeLock(retrievedlock)) if writeerror: raise writeerror diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 29a1d345..0ef337eb 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -355,17 +355,26 @@ def readfile(endpoint, filepath, userid, _lockid): yield chunk -def writefile(endpoint, filepath, userid, content, _lockid, islock=False): +def writefile(endpoint, filepath, userid, content, lockmd, islock=False): '''Write a file via xroot on behalf of the given userid. The entire content is written and any pre-existing file is deleted (or moved to the previous version if supported). With islock=True, the write explicitly disables versioning, and the file is opened with O_CREAT|O_EXCL, preventing race conditions.''' size = len(content) log.debug('msg="Invoking writeFile" filepath="%s" userid="%s" size="%d" islock="%s"' % (filepath, userid, size, islock)) - existingLock = getlock(endpoint, filepath, userid) + if lockmd: + appname, _ = lockmd + else: + appname = '' f = XrdClient.File() tstart = time.time() - rc, _ = f.open(_geturlfor(endpoint) + '/' + homepath + filepath + _eosargs(userid, not islock, size), + if islock: + # this is required to trigger the O_EXCL behavior on EOS when creating lock files + appname = 'fuse::wopi' + else: + # this is exclusively used to validate the lock with the app as holder, according to EOS specs (cf. _geneoslock()) + appname = 'wopi_' + appname.replace(' ', '_').lower() + rc, _ = f.open(_geturlfor(endpoint) + '/' + homepath + filepath + _eosargs(userid, appname, size), OpenFlags.NEW if islock else OpenFlags.DELETE, timeout=timeout) tend = time.time() if not rc.ok: @@ -376,6 +385,9 @@ def writefile(endpoint, filepath, userid, content, _lockid, islock=False): # any other failure is reported as is log.error('msg="Error opening the file for write" filepath="%s" elapsedTimems="%.1f" error="%s"' % (filepath, (tend-tstart)*1000, rc.message.strip('\n'))) + if 'file has a valid extended attribute lock' in rc.message: + raise IOError(common.LOCK_MISMATCH_ERROR) + # any other failure is reported as is raise IOError(rc.message.strip('\n')) rc, _ = f.write(content, offset=0, size=size) if not rc.ok: From f310d06b8e6deb4b77576548f0e4021d19bf1d69 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 18 Oct 2022 17:17:43 +0200 Subject: [PATCH 088/325] Added more lock-related tests In particular writefile is checked against lock mismatch, whereas readfile must work without lock (yet) --- test/test_storageiface.py | 79 ++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/test/test_storageiface.py b/test/test_storageiface.py index b8ee979c..4cd655b1 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -215,17 +215,17 @@ def test_lock(self): self.storage.writefile(self.endpoint, self.homepath + '/testlock', self.userid, databuf, None) statInfo = self.storage.stat(self.endpoint, self.homepath + '/testlock', self.userid) self.assertIsInstance(statInfo, dict) - self.storage.setlock(self.endpoint, self.homepath + '/testlock', self.userid, 'myapp', 'testlock') + self.storage.setlock(self.endpoint, self.homepath + '/testlock', self.userid, 'test app', 'testlock') l = self.storage.getlock(self.endpoint, self.homepath + '/testlock', self.userid) # noqa: E741 self.assertIsInstance(l, dict) self.assertEqual(l['lock_id'], 'testlock') - self.assertEqual(l['app_name'], 'myapp') + self.assertEqual(l['app_name'], 'test app') self.assertIsInstance(l['expiration'], dict) self.assertIsInstance(l['expiration']['seconds'], int) with self.assertRaises(IOError) as context: - self.storage.setlock(self.endpoint, self.homepath + '/testlock', self.userid, 'myapp', 'testlock2') + self.storage.setlock(self.endpoint, self.homepath + '/testlock', self.userid, 'test app', 'lockmismatch') self.assertIn(EXCL_ERROR, str(context.exception)) - self.storage.unlock(self.endpoint, self.homepath + '/testlock', self.userid, 'myapp', 'testlock') + self.storage.unlock(self.endpoint, self.homepath + '/testlock', self.userid, 'test app', 'testlock') self.storage.removefile(self.endpoint, self.homepath + '/testlock', self.userid) def test_refresh_lock(self): @@ -238,22 +238,23 @@ def test_refresh_lock(self): statInfo = self.storage.stat(self.endpoint, self.homepath + '/testrlock', self.userid) self.assertIsInstance(statInfo, dict) with self.assertRaises(IOError) as context: - self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'testlock') + self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'test app', 'testlock') self.assertIn('File was not locked', str(context.exception)) - self.storage.setlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'testlock') + self.storage.setlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'test app', 'testlock') with self.assertRaises(IOError) as context: - self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'newlock', 'mismatched') + self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'test app', 'newlock', 'mismatch') self.assertIn(LOCK_MISMATCH_ERROR, str(context.exception)) - self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp', 'testlock2', 'testlock') + self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'test app', 'newlock', 'testlock') l = self.storage.getlock(self.endpoint, self.homepath + '/testrlock', self.userid) # noqa: E741 self.assertIsInstance(l, dict) - self.assertEqual(l['lock_id'], 'testlock2') - self.assertEqual(l['app_name'], 'myapp') + self.assertEqual(l['lock_id'], 'newlock') + self.assertEqual(l['app_name'], 'test app') self.assertIsInstance(l['expiration'], dict) self.assertIsInstance(l['expiration']['seconds'], int) + self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'test app', 'newlock') with self.assertRaises(IOError) as context: - self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'myapp2', 'testlock2') - self.assertIn('File is locked by myapp', str(context.exception)) + self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'mismatched app', 'newlock') + self.assertIn('File is locked by test app', str(context.exception)) self.storage.removefile(self.endpoint, self.homepath + '/testrlock', self.userid) def test_lock_race(self): @@ -266,11 +267,11 @@ def test_lock_race(self): statInfo = self.storage.stat(self.endpoint, self.homepath + '/testlockrace', self.userid) self.assertIsInstance(statInfo, dict) t = Thread(target=self.storage.setlock, - args=[self.endpoint, self.homepath + '/testlockrace', self.userid, 'myapp', 'testlock']) + args=[self.endpoint, self.homepath + '/testlockrace', self.userid, 'test app', 'testlock']) t.start() with self.assertRaises(IOError) as context: time.sleep(0.001) - self.storage.setlock(self.endpoint, self.homepath + '/testlockrace', self.userid, 'myapp', 'testlock2') + self.storage.setlock(self.endpoint, self.homepath + '/testlockrace', self.userid, 'test app', 'testlock2') self.assertIn(EXCL_ERROR, str(context.exception)) self.storage.removefile(self.endpoint, self.homepath + '/testlockrace', self.userid) @@ -283,20 +284,26 @@ def test_lock_operations(self): self.storage.writefile(self.endpoint, self.homepath + '/testlockop', self.userid, databuf, None) statInfo = self.storage.stat(self.endpoint, self.homepath + '/testlockop', self.userid) self.assertIsInstance(statInfo, dict) - self.storage.setlock(self.endpoint, self.homepath + '/testlockop', self.userid, 'myapp', 'testlock') - self.storage.writefile(self.endpoint, self.homepath + '/testlockop', self.userid, databuf, 'testlock') - self.storage.setxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', 123, 'testlock') - self.storage.renamefile(self.endpoint, self.homepath + '/testlockop', self.homepath + '/testlockop_renamed', - self.userid, 'testlock') - self.storage.refreshlock(self.endpoint, self.homepath + '/testlockop_renamed', self.userid, 'myapp', 'testlock') + self.storage.setlock(self.endpoint, self.homepath + '/testlockop', self.userid, 'test app', 'testlock') + self.storage.writefile(self.endpoint, self.homepath + '/testlockop', self.userid, databuf, ('test app', 'testlock')) with self.assertRaises(IOError): - self.storage.writefile(self.endpoint, self.homepath + '/testlockop_renamed', self.userid, databuf, None) + # TODO different interfaces raise exceptions on either mismatching app xor mismatching lock payload, + # this is why we test that both mismatch. Could be improved, though we specifically care about the lock paylaod. + self.storage.writefile(self.endpoint, self.homepath + '/testlockop', self.userid, databuf, + ('mismatchapp', 'mismatchlock')) + self.storage.refreshlock(self.endpoint, self.homepath + '/testlockop', self.userid, 'test app', 'testlock') with self.assertRaises(IOError): - self.storage.setxattr(self.endpoint, self.homepath + '/testlockop_renamed', self.userid, 'testkey', 123, None) + self.storage.writefile(self.endpoint, self.homepath + '/testlockop', self.userid, databuf, None) with self.assertRaises(IOError): - self.storage.renamefile(self.endpoint, self.homepath + '/testlockop_renamed', self.homepath + '/testlockop', + self.storage.setxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', 123, None) + with self.assertRaises(IOError): + self.storage.rmxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', None) + with self.assertRaises(IOError): + self.storage.renamefile(self.endpoint, self.homepath + '/testlockop', self.homepath + '/testlockop_renamed', self.userid, None) - self.storage.removefile(self.endpoint, self.homepath + '/testlockop_renamed', self.userid) + for chunk in self.storage.readfile(self.endpoint, self.homepath + '/testlockop', self.userid, None): + self.assertNotIsInstance(chunk, IOError, 'raised by storage.readfile, lock shall be shared') + self.storage.removefile(self.endpoint, self.homepath + '/testlockop', self.userid) def test_expired_locks(self): '''Test lock operations on expired locks''' @@ -307,24 +314,24 @@ def test_expired_locks(self): self.storage.writefile(self.endpoint, self.homepath + '/testelock', self.userid, databuf, None) statInfo = self.storage.stat(self.endpoint, self.homepath + '/testelock', self.userid) self.assertIsInstance(statInfo, dict) - self.storage.setlock(self.endpoint, self.homepath + '/testelock', self.userid, 'myapp', 'testlock') + self.storage.setlock(self.endpoint, self.homepath + '/testelock', self.userid, 'test app', 'testlock') time.sleep(2.1) l = self.storage.getlock(self.endpoint, self.homepath + '/testelock', self.userid) # noqa: E741 self.assertEqual(l, None) - self.storage.setlock(self.endpoint, self.homepath + '/testelock', self.userid, 'myapp', 'testlock2') + self.storage.setlock(self.endpoint, self.homepath + '/testelock', self.userid, 'test app', 'testlock2') time.sleep(2.1) - self.storage.setlock(self.endpoint, self.homepath + '/testelock', self.userid, 'myapp', 'testlock3') + self.storage.setlock(self.endpoint, self.homepath + '/testelock', self.userid, 'test app', 'testlock3') l = self.storage.getlock(self.endpoint, self.homepath + '/testelock', self.userid) # noqa: E741 self.assertIsInstance(l, dict) self.assertEqual(l['lock_id'], 'testlock3') time.sleep(2.1) with self.assertRaises(IOError) as context: - self.storage.refreshlock(self.endpoint, self.homepath + '/testelock', self.userid, 'myapp', 'testlock4') + self.storage.refreshlock(self.endpoint, self.homepath + '/testelock', self.userid, 'test app', 'testlock4') self.assertIn('File was not locked', str(context.exception)) - self.storage.setlock(self.endpoint, self.homepath + '/testelock', self.userid, 'myapp', 'testlock5') + self.storage.setlock(self.endpoint, self.homepath + '/testelock', self.userid, 'test app', 'testlock5') time.sleep(2.1) with self.assertRaises(IOError) as context: - self.storage.unlock(self.endpoint, self.homepath + '/testelock', self.userid, 'myapp', 'testlock5') + self.storage.unlock(self.endpoint, self.homepath + '/testelock', self.userid, 'test app', 'testlock5') self.assertIn('File was not locked', str(context.exception)) self.storage.removefile(self.endpoint, self.homepath + '/testelock', self.userid) @@ -337,10 +344,11 @@ def test_remove_nofile(self): def test_xattr(self): '''Test all xattr methods with special chars''' self.storage.writefile(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, databuf, None) - self.storage.setxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 123, None) + self.storage.setlock(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'test app', 'xattrlock') + self.storage.setxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 123, 'xattrlock') v = self.storage.getxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey') self.assertEqual(v, '123') - self.storage.rmxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', None) + self.storage.rmxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 'xattrlock') v = self.storage.getxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey') self.assertEqual(v, None) self.storage.removefile(self.endpoint, self.homepath + '/test&xattr.txt', self.userid) @@ -348,10 +356,13 @@ def test_xattr(self): def test_rename_statx(self): '''Test renaming and statx of a file with special chars''' self.storage.writefile(self.endpoint, self.homepath + '/test.txt', self.userid, databuf, None) - self.storage.renamefile(self.endpoint, self.homepath + '/test.txt', self.homepath + '/test&ren.txt', self.userid, None) + self.storage.setlock(self.endpoint, self.homepath + '/test.txt', self.userid, 'test app', 'renamelock') + self.storage.renamefile(self.endpoint, self.homepath + '/test.txt', self.homepath + '/test&ren.txt', + self.userid, 'renamelock') statInfo = self.storage.statx(self.endpoint, self.homepath + '/test&ren.txt', self.userid) self.assertEqual(statInfo['filepath'], self.homepath + '/test&ren.txt') - self.storage.renamefile(self.endpoint, self.homepath + '/test&ren.txt', self.homepath + '/test.txt', self.userid, None) + self.storage.renamefile(self.endpoint, self.homepath + '/test&ren.txt', self.homepath + '/test.txt', + self.userid, 'renamelock') statInfo = self.storage.statx(self.endpoint, self.homepath + '/test.txt', self.userid) self.assertEqual(statInfo['filepath'], self.homepath + '/test.txt') self.storage.removefile(self.endpoint, self.homepath + '/test.txt', self.userid) From 01a71388c8b8b050d42c22ee0c97bd8b8d8d1bfa Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 19 Oct 2022 11:05:19 +0200 Subject: [PATCH 089/325] Refactored validatelock and used in localiface for all lock validations --- src/core/commoniface.py | 18 ++++++++-------- src/core/localiface.py | 47 ++++++++++++++++++----------------------- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/src/core/commoniface.py b/src/core/commoniface.py index 6eeffaea..8f80ea5e 100644 --- a/src/core/commoniface.py +++ b/src/core/commoniface.py @@ -17,7 +17,7 @@ ENOENT_MSG = 'No such file or directory' # standard error thrown when attempting to overwrite a file/xattr in O_EXCL mode -EXCL_ERROR = 'File exists and islock flag requested' +EXCL_ERROR = 'File/xattr exists but EXCL mode requested' # error thrown on refreshlock when the payload does not match LOCK_MISMATCH_ERROR = 'Existing lock payload does not match' @@ -92,18 +92,18 @@ def decodeinode(inode): return e, urlsafe_b64decode(f.encode()).decode() -def validatelock(filepath, appname, oldlock, oldvalue, op, log): +def validatelock(filepath, currlock, appname, value, op, log): '''Common logic for validating locks in the xrootd and local storage interfaces. Duplicates some logic implemented in Reva for the cs3 storage interface''' try: - if not oldlock: + if not currlock: raise IOError('File was not locked or lock had expired') - if oldvalue and oldlock['lock_id'] != oldvalue: + if appname and currlock['app_name'] != appname \ + and currlock['app_name'] != 'wopi' and appname != 'wopi': # TODO deprecated, to be removed after CERNBox rollout + raise IOError('File is locked by %s' % currlock['app_name']) + if value != currlock['lock_id']: raise IOError(LOCK_MISMATCH_ERROR) - if appname and oldlock['app_name'] != appname \ - and oldlock['app_name'] != 'wopi' and appname != 'wopi': # TODO deprecated, to be removed after CERNBox rollout - raise IOError('File is locked by %s' % oldlock['app_name']) except IOError as e: - log.warning('msg="Failed to %s" filepath="%s" appname="%s" reason="%s"' % - (op, filepath, appname, e)) + log.warning('msg="Failed to %s" filepath="%s" appname="%s" lockid="%s" currlock="%s" reason="%s"' % + (op, filepath, appname, value, currlock, e)) raise diff --git a/src/core/localiface.py b/src/core/localiface.py index c3530f0e..582b6f8d 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -22,9 +22,6 @@ log = None homepath = None -# a conventional value used by _checklock() -LOCK = '__LOCK__' - class Flock: '''A simple class to lock/unlock when entering/leaving a runtime context @@ -105,20 +102,10 @@ def statx(endpoint, filepath, userid, versioninv=1): return stat(endpoint, filepath, userid) -def _checklock(op, endpoint, filepath, userid, lockid): - '''Verify if the given lockid matches the existing one on the given filepath, if any''' - if lockid == LOCK: - # this is a special value to skip the check, used by the lock operations themselves - return - lock = getlock(endpoint, filepath, userid) - if lock and lock['lock_id'] != lockid: # here we could also validate the app/holder - log.warning('msg="%s: file was locked" filepath="%s" holder="%s"' % (op, filepath, lock['app_name'])) - raise IOError(common.LOCK_MISMATCH_ERROR) - - def setxattr(endpoint, filepath, userid, key, value, lockid): '''Set the extended attribute to on behalf of the given userid''' - _checklock('setxattr', endpoint, filepath, userid, lockid) + if key != common.LOCKKEY: + common.validatelock(filepath, getlock(endpoint, filepath, userid), None, lockid, 'setxattr', log) try: os.setxattr(_getfilepath(filepath), 'user.' + key, str(value).encode()) except OSError as e: @@ -137,7 +124,8 @@ def getxattr(_endpoint, filepath, _userid, key): def rmxattr(endpoint, filepath, userid, key, lockid): '''Remove the extended attribute on behalf of the given userid''' - _checklock('rmxattr', endpoint, filepath, userid, lockid) + if key != common.LOCKKEY: + common.validatelock(filepath, getlock(endpoint, filepath, userid), None, lockid, 'rmxattr', log) try: os.removexattr(_getfilepath(filepath), 'user.' + key) except OSError as e: @@ -153,7 +141,8 @@ def setlock(endpoint, filepath, userid, appname, value): try: with fl: if not getlock(endpoint, filepath, userid): - setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value), LOCK) + log.debug('msg="setlock: invoking setxattr" filepath="%s" value="%s"' % (filepath, value)) + setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value), None) else: raise IOError(common.EXCL_ERROR) except BlockingIOError as e: @@ -171,23 +160,27 @@ def getlock(endpoint, filepath, _userid): return lock # otherwise, the lock had expired: drop it and return None log.debug('msg="getlock: removed stale lock" filepath="%s"' % filepath) - rmxattr(endpoint, filepath, '0:0', common.LOCKKEY, LOCK) + rmxattr(endpoint, filepath, '0:0', common.LOCKKEY, None) return None def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): '''Refresh the lock value as an xattr on behalf of the given userid''' - common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), oldvalue, 'refreshlock', log) + currlock = getlock(endpoint, filepath, userid) + if not oldvalue and currlock: + # this is a pure refresh operation + oldvalue = currlock['lock_id'] + common.validatelock(filepath, currlock, appname, oldvalue, 'refreshlock', log) # this is non-atomic, but if we get here the lock was already held log.debug('msg="Invoked refreshlock" filepath="%s" value="%s"' % (filepath, value)) - setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value), LOCK) + setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value), None) def unlock(endpoint, filepath, userid, appname, value): '''Remove the lock as an xattr on behalf of the given userid''' - common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), None, 'unlock', log) + common.validatelock(filepath, getlock(endpoint, filepath, userid), appname, value, 'unlock', log) log.debug('msg="Invoked unlock" filepath="%s" value="%s"' % (filepath, value)) - rmxattr(endpoint, filepath, '0:0', common.LOCKKEY, LOCK) + rmxattr(endpoint, filepath, '0:0', common.LOCKKEY, None) def readfile(_endpoint, filepath, _userid, _lockid): @@ -221,10 +214,10 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): content = bytes(content, 'UTF-8') size = len(content) if lockmd: - _, lockid = lockmd - else: - lockid = None - _checklock('writefile', endpoint, filepath, userid, lockid) + appname, lockid = lockmd + common.validatelock(filepath, getlock(endpoint, filepath, userid), appname, lockid, 'writefile', log) + elif getlock(endpoint, filepath, userid): + raise IOError(common.LOCK_MISMATCH_ERROR) log.debug('msg="Invoking writeFile" filepath="%s" size="%d"' % (filepath, size)) tstart = time.time() if islock: @@ -261,7 +254,7 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): def renamefile(endpoint, origfilepath, newfilepath, userid, lockid): '''Rename a file from origfilepath to newfilepath on behalf of the given userid.''' - _checklock('renamefile', endpoint, origfilepath, userid, lockid) + common.validatelock(origfilepath, getlock(endpoint, origfilepath, userid), None, lockid, 'renamefile', log) try: os.rename(_getfilepath(origfilepath), _getfilepath(newfilepath)) except OSError as e: From 11a895eced4be1761429aa24bf50933d8c0e6c42 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 19 Oct 2022 14:51:18 +0200 Subject: [PATCH 090/325] One more artifact folder to ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 43c6e740..23b6e53c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *pyc *rpm +.cache .mypy_cache From b28cc14affcb7b27b0cd69a11741bde14210d725 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 19 Oct 2022 14:53:45 +0200 Subject: [PATCH 091/325] xroot: implement EOS-supported locking and removed obsoleted code --- src/core/xrootiface.py | 65 +++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 0ef337eb..758b45bf 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -17,6 +17,7 @@ EOSVERSIONPREFIX = '.sys.v#.' EXCL_XATTR_MSG = 'exclusive set for existing attribute' OK_MSG = '[SUCCESS]' # this is what xroot returns on success +EOSLOCKKEY = 'sys.app.lock' # module-wide state config = None @@ -73,7 +74,7 @@ def _geturlfor(endpoint): return endpoint if endpoint.find('root://') == 0 else ('root://' + endpoint.replace('newproject', 'eosproject') + '.cern.ch') -def _eosargs(userid, atomicwrite=0, bookingsize=0): +def _eosargs(userid, app='wopi', bookingsize=0): '''Assume userid is in the form uid:gid and split it into uid, gid plus generate extra EOS-specific arguments for the xroot URL''' try: @@ -83,12 +84,18 @@ def _eosargs(userid, atomicwrite=0, bookingsize=0): raise ValueError ruid = int(userid[0]) rgid = int(userid[1]) - return '?eos.ruid=%d&eos.rgid=%d' % (ruid, rgid) + '&eos.app=' + ('fuse::wopi' if not atomicwrite else 'wopi') + \ + return '?eos.ruid=%d&eos.rgid=%d' % (ruid, rgid) + '&eos.app=' + app + \ (('&eos.bookingsize=' + str(bookingsize)) if bookingsize else '') except (ValueError, IndexError): raise ValueError('Only Unix-based userid is supported with xrootd storage: %s' % userid) +def _geneoslock(appname): + '''One-liner to generate an EOS app lock. Type is `shared` (hardcoded) for WOPI apps, `exclusive` is also supported''' + return 'expires:%d,type:shared,owner:*:wopi_%s' % \ + (int(time.time()) + config.getint("general", "wopilockexpiration"), appname.replace(' ', '_').lower()) + + def _xrootcmd(endpoint, cmd, subcmd, userid, args): '''Perform the / action on the special /proc/user path on behalf of the given userid. Note that this is entirely EOS-specific.''' @@ -253,7 +260,7 @@ def statx(endpoint, fileref, userid, versioninv=1): } -def setxattr(endpoint, filepath, _userid, key, value, _lockid): +def setxattr(endpoint, filepath, userid, key, value, lockid): '''Set the extended attribute to via a special open. The userid is overridden to make sure it also works on shared files.''' _xrootcmd(endpoint, 'attr', 'set', '0:0', 'mgm.attr.key=user.' + key + '&mgm.attr.value=' + str(value) @@ -263,19 +270,27 @@ def setxattr(endpoint, filepath, _userid, key, value, _lockid): def getxattr(endpoint, filepath, _userid, key): '''Get the extended attribute via a special open. The userid is overridden to make sure it also works on shared files.''' + if 'user' not in key and 'sys' not in key: + # if nothing is given, assume it's a user attr + key = 'user.' + key try: res = _xrootcmd(endpoint, 'attr', 'get', '0:0', - 'mgm.attr.key=user.' + key + '&mgm.path=' + _getfilepath(filepath, encodeamp=True)) + 'mgm.attr.key=' + key + '&mgm.path=' + _getfilepath(filepath, encodeamp=True)) # if no error, the response comes in the format ="" return res.split('"')[1] except (IndexError, IOError): return None -def rmxattr(endpoint, filepath, _userid, key, _lockid): +def rmxattr(endpoint, filepath, userid, key, lockid): '''Remove the extended attribute via a special open. The userid is overridden to make sure it also works on shared files.''' - _xrootcmd(endpoint, 'attr', 'rm', '0:0', 'mgm.attr.key=user.' + key + '&mgm.path=' + _getfilepath(filepath, encodeamp=True)) + if key not in (EOSLOCKKEY, common.LOCKKEY): + common.validatelock(filepath, getlock(endpoint, filepath, userid), None, lockid, 'rmxattr', log) + if 'user' not in key and 'sys' not in key: + # if nothing is given, assume it's a user attr + key = 'user.' + key + _xrootcmd(endpoint, 'attr', 'rm', '0:0', 'mgm.attr.key=' + key + '&mgm.path=' + _getfilepath(filepath, encodeamp=True)) def setlock(endpoint, filepath, userid, appname, value, recurse=False): @@ -283,13 +298,13 @@ def setlock(endpoint, filepath, userid, appname, value, recurse=False): The special option "c" (create-if-not-exists) is used to be atomic''' try: log.debug('msg="Invoked setlock" filepath="%s" value="%s"' % (filepath, value)) - setxattr(endpoint, filepath, userid, common.LOCKKEY, - common.genrevalock(appname, value) + '&mgm.option=c', None) + setxattr(endpoint, filepath, userid, EOSLOCKKEY, _geneoslock(appname) + '&mgm.option=c', None) + setxattr(endpoint, filepath, userid, common.LOCKKEY, common.genrevalock(appname, value), None) except IOError as e: - # TODO need to confirm this error message once EOS-5145 is implemented - if EXCL_XATTR_MSG in str(e) or 'flock already held' in str(e): + if EXCL_XATTR_MSG in str(e): # check for pre-existing stale locks (this is now not atomic) if not getlock(endpoint, filepath, userid) and not recurse: + rmxattr(endpoint, filepath, userid, EOSLOCKKEY, None) setlock(endpoint, filepath, userid, appname, value, recurse=True) else: # the lock is valid, raise conflict error @@ -310,22 +325,28 @@ def getlock(endpoint, filepath, userid): # otherwise, the lock had expired: drop it and return None log.debug('msg="getlock: removed stale lock" filepath="%s"' % filepath) rmxattr(endpoint, filepath, userid, common.LOCKKEY, None) - return None # no pre-existing lock found, or error attempting to read it: assume it does not exist + return None def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): '''Refresh the lock value as an xattr''' - common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), oldvalue, 'refreshlock', log) + currlock = getlock(endpoint, filepath, userid) + if not oldvalue and currlock: + # this is a pure refresh operation + oldvalue = currlock['lock_id'] + common.validatelock(filepath, currlock, appname, oldvalue, 'refreshlock', log) log.debug('msg="Invoked refreshlock" filepath="%s" value="%s"' % (filepath, value)) # this is non-atomic, but the lock was already held + setxattr(endpoint, filepath, userid, EOSLOCKKEY, _geneoslock(appname), None) setxattr(endpoint, filepath, userid, common.LOCKKEY, common.genrevalock(appname, value), None) def unlock(endpoint, filepath, userid, appname, value): '''Remove a lock as an xattr''' - common.validatelock(filepath, appname, getlock(endpoint, filepath, userid), None, 'unlock', log) + common.validatelock(filepath, getlock(endpoint, filepath, userid), appname, value, 'unlock', log) log.debug('msg="Invoked unlock" filepath="%s" value="%s"' % (filepath, value)) rmxattr(endpoint, filepath, userid, common.LOCKKEY, None) + rmxattr(endpoint, filepath, userid, EOSLOCKKEY, None) def readfile(endpoint, filepath, userid, _lockid): @@ -382,12 +403,12 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): # racing against an existing file log.info('msg="File exists on write but islock flag requested" filepath="%s"' % filepath) raise IOError(common.EXCL_ERROR) - # any other failure is reported as is - log.error('msg="Error opening the file for write" filepath="%s" elapsedTimems="%.1f" error="%s"' % - (filepath, (tend-tstart)*1000, rc.message.strip('\n'))) if 'file has a valid extended attribute lock' in rc.message: + log.warning('msg="Lock mismatch when writing file" filepath="%s"' % filepath) raise IOError(common.LOCK_MISMATCH_ERROR) # any other failure is reported as is + log.error('msg="Error opening the file for write" filepath="%s" elapsedTimems="%.1f" error="%s"' % + (filepath, (tend-tstart)*1000, rc.message.strip('\n'))) raise IOError(rc.message.strip('\n')) rc, _ = f.write(content, offset=0, size=size) if not rc.ok: @@ -404,21 +425,13 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): log.error('msg="Error closing the file" filepath="%s" elapsedTimems="%.1f" error="%s"' % (filepath, (tend-tstart)*1000, rc.message.strip('\n'))) raise IOError(rc.message.strip('\n')) - if existingLock: - try: - setlock(endpoint, filepath, userid, existingLock['app_name'], existingLock['lock_id'], False) - except IOError as e: - if str(e) == common.EXCL_ERROR: - # new EOS versions do preserve the attributes, so this would fail but it's OK - pass - else: - raise log.info('msg="File written successfully" filepath="%s" elapsedTimems="%.1f" islock="%s"' % (filepath, (tend-tstart)*1000, islock)) -def renamefile(endpoint, origfilepath, newfilepath, userid, _lockid): +def renamefile(endpoint, origfilepath, newfilepath, userid, lockid): '''Rename a file via a special open from origfilepath to newfilepath on behalf of the given userid.''' + common.validatelock(origfilepath, getlock(endpoint, origfilepath, userid), None, lockid, 'renamefile', log) _xrootcmd(endpoint, 'file', 'rename', userid, 'mgm.path=' + _getfilepath(origfilepath, encodeamp=True) + '&mgm.file.source=' + _getfilepath(origfilepath, encodeamp=True) + '&mgm.file.target=' + _getfilepath(newfilepath, encodeamp=True)) From 666a4088077f7e43e2f5e5d209ea162ffd2a2263 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 19 Oct 2022 14:54:04 +0200 Subject: [PATCH 092/325] Relaxed setxattr when there is no lock and lockid is None --- src/core/localiface.py | 5 ++++- src/core/xrootiface.py | 11 ++++++++++- test/test_storageiface.py | 3 +++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/core/localiface.py b/src/core/localiface.py index 582b6f8d..bff3f029 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -105,7 +105,10 @@ def statx(endpoint, filepath, userid, versioninv=1): def setxattr(endpoint, filepath, userid, key, value, lockid): '''Set the extended attribute to on behalf of the given userid''' if key != common.LOCKKEY: - common.validatelock(filepath, getlock(endpoint, filepath, userid), None, lockid, 'setxattr', log) + currlock = getlock(endpoint, filepath, userid) + if currlock: + # enforce lock only if previously set + common.validatelock(filepath, currlock, None, lockid, 'setxattr', log) try: os.setxattr(_getfilepath(filepath), 'user.' + key, str(value).encode()) except OSError as e: diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 758b45bf..f4316c8e 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -263,7 +263,16 @@ def statx(endpoint, fileref, userid, versioninv=1): def setxattr(endpoint, filepath, userid, key, value, lockid): '''Set the extended attribute to via a special open. The userid is overridden to make sure it also works on shared files.''' - _xrootcmd(endpoint, 'attr', 'set', '0:0', 'mgm.attr.key=user.' + key + '&mgm.attr.value=' + str(value) + if key not in (EOSLOCKKEY, common.LOCKKEY): + currlock = getlock(endpoint, filepath, userid) + if currlock: + # enforce lock only if previously set + common.validatelock(filepath, currlock, None, lockid, 'setxattr', log) + # else skip the check as we're setting the lock itself + if 'user' not in key and 'sys' not in key: + # if nothing is given, assume it's a user attr + key = 'user.' + key + _xrootcmd(endpoint, 'attr', 'set', '0:0', 'mgm.attr.key=' + key + '&mgm.attr.value=' + str(value) + '&mgm.path=' + _getfilepath(filepath, encodeamp=True)) diff --git a/test/test_storageiface.py b/test/test_storageiface.py index 4cd655b1..b1931ed0 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -294,6 +294,8 @@ def test_lock_operations(self): self.storage.refreshlock(self.endpoint, self.homepath + '/testlockop', self.userid, 'test app', 'testlock') with self.assertRaises(IOError): self.storage.writefile(self.endpoint, self.homepath + '/testlockop', self.userid, databuf, None) + with self.assertRaises(IOError): + self.storage.setxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', 123, 'mismatchlock') with self.assertRaises(IOError): self.storage.setxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', 123, None) with self.assertRaises(IOError): @@ -344,6 +346,7 @@ def test_remove_nofile(self): def test_xattr(self): '''Test all xattr methods with special chars''' self.storage.writefile(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, databuf, None) + self.storage.setxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 123, None) self.storage.setlock(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'test app', 'xattrlock') self.storage.setxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 123, 'xattrlock') v = self.storage.getxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey') From dbcafcc8c46ae6f3c89b28e7bf8b81518aa62fa2 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 29 Oct 2022 08:41:45 +0200 Subject: [PATCH 093/325] Test: get pwd from the terminal, not from config Plus some adaptation to CS3 output and removal of a CS3-related obsoleted check --- test/test_storageiface.py | 23 ++++++++++------------- test/wopiserver-test.conf | 3 +-- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/test/test_storageiface.py b/test/test_storageiface.py index b1931ed0..85033815 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -15,6 +15,7 @@ import os import time from threading import Thread +from getpass import getpass sys.path.append('src') # for tests out of the git repo sys.path.append('/app') # for tests within the Docker image from core.commoniface import EXCL_ERROR, LOCK_MISMATCH_ERROR, ENOENT_MSG # noqa: E402 @@ -61,7 +62,8 @@ def globalinit(cls): if 'cs3' in storagetype: # we need to login for this case cls.username = cls.userid - cls.userid = cls.storage.authenticate_for_test(cls.userid, config.get('cs3', 'userpwd')) + pwd = getpass("Please type %s's password to access the storage: " % cls.username) + cls.userid = cls.storage.authenticate_for_test(cls.username, pwd) cls.homepath = config.get('cs3', 'storagehomepath') except ImportError: print("Missing module when attempting to import %s. Please make sure dependencies are met." % storagetype) @@ -99,11 +101,6 @@ def test_statx_fileid(self): self.assertTrue('size' in statInfo, 'Missing size from stat output') self.assertTrue('mtime' in statInfo, 'Missing mtime from stat output') self.assertTrue('etag' in statInfo, 'Missing etag from stat output') - if self.endpoint in str(statInfo['inode']): - # detected CS3 storage, test if fileid-based stat is supported - # (notably, homepath is not part of the fileid) - statInfoId = self.storage.stat(self.endpoint, 'fileid-' + self.username + '%2Ftest.txt', self.userid) - self.assertEqual(statInfo['inode'], statInfoId['inode']) self.storage.removefile(self.endpoint, self.homepath + '/test.txt', self.userid) def test_statx_invariant_fileid(self): @@ -359,15 +356,15 @@ def test_xattr(self): def test_rename_statx(self): '''Test renaming and statx of a file with special chars''' self.storage.writefile(self.endpoint, self.homepath + '/test.txt', self.userid, databuf, None) - self.storage.setlock(self.endpoint, self.homepath + '/test.txt', self.userid, 'test app', 'renamelock') - self.storage.renamefile(self.endpoint, self.homepath + '/test.txt', self.homepath + '/test&ren.txt', - self.userid, 'renamelock') + statInfo = self.storage.statx(self.endpoint, self.homepath + '/test.txt', self.userid) + pathref = statInfo['filepath'][:statInfo['filepath'].rfind('/')] + + self.storage.renamefile(self.endpoint, self.homepath + '/test.txt', self.homepath + '/test&ren.txt', self.userid, None) statInfo = self.storage.statx(self.endpoint, self.homepath + '/test&ren.txt', self.userid) - self.assertEqual(statInfo['filepath'], self.homepath + '/test&ren.txt') - self.storage.renamefile(self.endpoint, self.homepath + '/test&ren.txt', self.homepath + '/test.txt', - self.userid, 'renamelock') + self.assertEqual(statInfo['filepath'], pathref + '/test&ren.txt') + self.storage.renamefile(self.endpoint, self.homepath + '/test&ren.txt', self.homepath + '/test.txt', self.userid, None) statInfo = self.storage.statx(self.endpoint, self.homepath + '/test.txt', self.userid) - self.assertEqual(statInfo['filepath'], self.homepath + '/test.txt') + self.assertEqual(statInfo['filepath'], pathref + '/test.txt') self.storage.removefile(self.endpoint, self.homepath + '/test.txt', self.userid) diff --git a/test/wopiserver-test.conf b/test/wopiserver-test.conf index ffa8faaf..656bf4cc 100644 --- a/test/wopiserver-test.conf +++ b/test/wopiserver-test.conf @@ -20,9 +20,8 @@ revagateway = cbox-ocisdev-01:9142 authtokenvalidity = 3600 sslverify = False userid = -userpwd = endpoint = -storagehomepath = +storagehomepath = [xroot] storageserver = root://eoshomecanary From e4d36ab6b6b7b854a620824a6df6b6da4a40dad1 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 29 Oct 2022 08:44:36 +0200 Subject: [PATCH 094/325] cs3iface: reintroduced absolute path support for testing purposes --- src/core/cs3iface.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index 649ec15c..fa8998a7 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -56,7 +56,10 @@ def _getcs3reference(endpoint, fileref): if len(parts) == 2: space_id = parts[1] - if fileref.find('/') > 0: + if fileref.find('/') == 0: + # assume we have an absolute path (works in Reva master, not in edge) + ref = cs3spr.Reference(path=fileref) + elif fileref.find('/') > 0: # assume we have a relative path in the form `/`, # also works if we get `//` ref = cs3spr.Reference(resource_id=cs3spr.ResourceId(storage_id=endpoint, space_id=space_id, @@ -72,7 +75,7 @@ def authenticate_for_test(userid, userpwd): '''Use basic authentication against Reva for testing purposes''' authReq = cs3gw.AuthenticateRequest(type='basic', client_id=userid, client_secret=userpwd) authRes = ctx['cs3gw'].Authenticate(authReq) - log.debug('msg="Authenticated user" res="%s"' % authRes) + log.debug('msg="Authenticated user" userid="%s"' % authRes.user.id) if authRes.status.code != cs3code.CODE_OK: raise IOError('Failed to authenticate as user ' + userid + ': ' + authRes.status.message) return authRes.token From 9fba6db4883e3198cc870e4f997dbc435d8be812 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 29 Oct 2022 08:45:00 +0200 Subject: [PATCH 095/325] cs3iface: fixed error reporting --- src/core/cs3iface.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index fa8998a7..034f7f03 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -367,10 +367,10 @@ def removefile(endpoint, filepath, userid, _force=False): req = cs3sp.DeleteRequest(ref=reference) res = ctx['cs3gw'].Delete(request=req, metadata=[('x-access-token', userid)]) if res.status.code != cs3code.CODE_OK: - if str(res) == common.ENOENT_MSG: + if 'path not found' in str(res): log.info('msg="Invoked removefile on non-existing file" filepath="%s"' % filepath) - else: - log.error('msg="Failed to remove file" filepath="%s" trace="%s" code="%s" reason="%s"' % - (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) + raise IOError(common.ENOENT_MSG) + log.error('msg="Failed to remove file" filepath="%s" trace="%s" code="%s" reason="%s"' % + (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) log.debug('msg="Invoked removefile" result="%s"' % res) From ceae8df85e439ac5064da93f55be5fdbe3f4e9b9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 19 Oct 2022 15:02:56 +0200 Subject: [PATCH 096/325] Changed handling of locks in rename operations Locks are typically NOT taken (nor passed) by MS Office 365 on renames, yet they need to be handled because the backend may implement a rename as a copy + delete --- src/core/cs3iface.py | 6 +++++- src/core/localiface.py | 3 +-- src/core/wopi.py | 24 +++++++++++++++++------- src/core/xrootiface.py | 3 +-- test/test_storageiface.py | 7 +++---- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index 034f7f03..78535ef3 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -317,7 +317,7 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): if res.status.code != cs3code.CODE_OK: log.error('msg="Failed to initiateFileUpload on write" filepath="%s" trace="%s" code="%s" reason="%s"' % (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) - if 'lock' in res.status.message: # TODO find the error code returned by Reva once this is implemented + if '_lock_' in res.status.message: # TODO find the error code returned by Reva once this is implemented raise IOError(common.LOCK_MISMATCH_ERROR) raise IOError(res.status.message) tend = time.time() @@ -353,6 +353,10 @@ def renamefile(endpoint, filepath, newfilepath, userid, lockid): req = cs3sp.MoveRequest(source=reference, destination=newfileref, lock_id=lockid) res = ctx['cs3gw'].Move(request=req, metadata=[('x-access-token', userid)]) + if res.status.code in [cs3code.CODE_FAILED_PRECONDITION, cs3code.CODE_ABORTED]: + log.info('msg="Failed precondition on rename" filepath="%s" trace="%s" reason="%s"' % + (filepath, res.status.trace, res.status.message.replace('"', "'"))) + raise IOError(common.EXCL_ERROR) if res.status.code != cs3code.CODE_OK: log.error('msg="Failed to rename file" filepath="%s" trace="%s" code="%s" reason="%s"' % (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) diff --git a/src/core/localiface.py b/src/core/localiface.py index bff3f029..8e54701e 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -255,9 +255,8 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): (filepath, (tend - tstart) * 1000, islock)) -def renamefile(endpoint, origfilepath, newfilepath, userid, lockid): +def renamefile(endpoint, origfilepath, newfilepath, _userid, _lockid): '''Rename a file from origfilepath to newfilepath on behalf of the given userid.''' - common.validatelock(origfilepath, getlock(endpoint, origfilepath, userid), None, lockid, 'renamefile', log) try: os.rename(_getfilepath(origfilepath), _getfilepath(newfilepath)) except OSError as e: diff --git a/src/core/wopi.py b/src/core/wopi.py index 914dfff1..e3510132 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -457,7 +457,7 @@ def renameFile(fileid, reqheaders, acctok): log.warning('msg="Missing argument" client="%s" requestedUrl="%s" error="%s" token="%s"' % (flask.request.remote_addr, flask.request.base_url, e, flask.request.args.get('access_token')[-20:])) return 'Missing argument', http.client.BAD_REQUEST - lock = reqheaders.get('X-WOPI-Lock') + lock = reqheaders.get('X-WOPI-Lock') # may not be specified retrievedLock, _ = utils.retrieveWopiLock(fileid, 'RENAMEFILE', lock, acctok) if retrievedLock is not None and not utils.compareWopiLocks(retrievedLock, lock): return utils.makeConflictResponse('RENAMEFILE', acctok['userid'], retrievedLock, lock, 'NA', @@ -468,7 +468,12 @@ def renameFile(fileid, reqheaders, acctok): + os.path.splitext(acctok['filename'])[1] if targetName.find('.') < 0 else '' log.info('msg="RenameFile" user="%s" filename="%s" token="%s" targetname="%s"' % (acctok['userid'][-20:], acctok['filename'], flask.request.args['access_token'][-20:], targetName)) - st.renamefile(acctok['endpoint'], acctok['filename'], targetName, acctok['userid'], utils.encodeLock(retrievedLock)) + + # try to rename and pass the lock if present. Note that WOPI specs do not require files to be locked + # on rename operations, but the backend may still fail as renames may be implemented as copy + delete, + # which may require to pass a lock. + st.renamefile(acctok['endpoint'], acctok['filename'], targetName, acctok['userid'], + utils.encodeLock(retrievedLock) if retrievedLock else None) # also rename the lock if applicable if os.path.splitext(acctok['filename'])[1] in srv.codetypes: st.renamefile(acctok['endpoint'], utils.getLibreOfficeLockName(acctok['filename']), @@ -476,12 +481,17 @@ def renameFile(fileid, reqheaders, acctok): # send the response as JSON return flask.Response(json.dumps(renamemd), mimetype='application/json') except IOError as e: - if common.ENOENT_MSG in str(e): - return 'File not found', http.client.NOT_FOUND - log.info('msg="RenameFile" token="%s" error="%s"' % (flask.request.args['access_token'][-20:], e)) + log.warn('msg="RenameFile" token="%s" error="%s"' % (flask.request.args['access_token'][-20:], e)) resp = flask.Response() - resp.headers['X-WOPI-InvalidFileNameError'] = 'Failed to rename: %s' % e - resp.status_code = http.client.BAD_REQUEST + if common.ENOENT_MSG in str(e): + resp.headers['X-WOPI-InvalidFileNameError'] = 'File not found' + resp.status_code = http.client.NOT_FOUND + elif common.EXCL_ERROR in str(e): + resp.headers['X-WOPI-InvalidFileNameError'] = 'Cannot rename/move unlocked file' + resp.status_code = http.client.NOT_IMPLEMENTED + else: + resp.headers['X-WOPI-InvalidFileNameError'] = 'Failed to rename: %s' % e + resp.status_code = http.client.INTERNAL_SERVER_ERROR return resp diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index f4316c8e..8c004f10 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -438,9 +438,8 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): (filepath, (tend-tstart)*1000, islock)) -def renamefile(endpoint, origfilepath, newfilepath, userid, lockid): +def renamefile(endpoint, origfilepath, newfilepath, userid, _lockid): '''Rename a file via a special open from origfilepath to newfilepath on behalf of the given userid.''' - common.validatelock(origfilepath, getlock(endpoint, origfilepath, userid), None, lockid, 'renamefile', log) _xrootcmd(endpoint, 'file', 'rename', userid, 'mgm.path=' + _getfilepath(origfilepath, encodeamp=True) + '&mgm.file.source=' + _getfilepath(origfilepath, encodeamp=True) + '&mgm.file.target=' + _getfilepath(newfilepath, encodeamp=True)) diff --git a/test/test_storageiface.py b/test/test_storageiface.py index 85033815..ba76fdc0 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -297,12 +297,11 @@ def test_lock_operations(self): self.storage.setxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', 123, None) with self.assertRaises(IOError): self.storage.rmxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', None) - with self.assertRaises(IOError): - self.storage.renamefile(self.endpoint, self.homepath + '/testlockop', self.homepath + '/testlockop_renamed', - self.userid, None) for chunk in self.storage.readfile(self.endpoint, self.homepath + '/testlockop', self.userid, None): self.assertNotIsInstance(chunk, IOError, 'raised by storage.readfile, lock shall be shared') - self.storage.removefile(self.endpoint, self.homepath + '/testlockop', self.userid) + self.storage.renamefile(self.endpoint, self.homepath + '/testlockop', self.homepath + '/testlockop_renamed', + self.userid, 'testlock') + self.storage.removefile(self.endpoint, self.homepath + '/testlockop_renamed', self.userid) def test_expired_locks(self): '''Test lock operations on expired locks''' From 3223ff6cbe8ef22a49827da453d5d6321aba8342 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 4 Nov 2022 12:59:57 +0100 Subject: [PATCH 097/325] Fixed PutRelative return codes in some corner cases Most notable cases are when the user has no permission or when the request is malformed --- src/core/wopi.py | 22 +++++++++++++--------- src/core/wopiutils.py | 6 ++++-- src/core/xrootiface.py | 3 +++ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index e3510132..b6bbd6a8 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -338,9 +338,9 @@ def putRelative(fileid, reqheaders, acctok): 'overwrite="%r" wopitimestamp="%s" token="%s"' % (acctok['userid'], acctok['filename'], fileid, suggTarget, relTarget, overwriteTarget, reqheaders.get('X-WOPI-TimeStamp'), flask.request.args['access_token'][-20:])) - # either one xor the other must be present; note we can't use `^` as we have a mix of str and NoneType + # either one xor the other MUST be present; note we can't use `^` as we have a mix of str and NoneType if (suggTarget and relTarget) or (not suggTarget and not relTarget): - return '', http.client.NOT_IMPLEMENTED + return 'Conflicting headers given', http.client.BAD_REQUEST if suggTarget: # the suggested target is a UTF7-encoded (!) filename that can be changed to avoid collisions suggTarget = suggTarget.encode().decode('utf-7') @@ -360,10 +360,10 @@ def putRelative(fileid, reqheaders, acctok): # OK, the targetName is good to go break # we got another error with this file, fail - log.warning('msg="PutRelative" user="%s" filename="%s" token="%s" suggTarget="%s" error="%s"' % - (acctok['userid'][-20:], targetName, flask.request.args['access_token'][-20:], - suggTarget, str(e))) - return '', http.client.BAD_REQUEST + log.error('msg="PutRelative" user="%s" filename="%s" token="%s" suggTarget="%s" error="%s"' % + (acctok['userid'][-20:], targetName, flask.request.args['access_token'][-20:], + suggTarget, str(e))) + return 'Error with the given target', http.client.INTERNAL_SERVER_ERROR else: # the relative target is a UTF7-encoded filename to be respected, and that may overwrite an existing file relTarget = os.path.dirname(acctok['filename']) + os.path.sep + relTarget.encode().decode('utf-7') # make full path @@ -374,13 +374,14 @@ def putRelative(fileid, reqheaders, acctok): retrievedTargetLock, _ = utils.retrieveWopiLock(fileid, 'PUT_RELATIVE', None, acctok, overridefn=relTarget) # deny if lock is valid or if overwriteTarget is False if not overwriteTarget or retrievedTargetLock: - return utils.makeConflictResponse('PUT_RELATIVE', acctok['userid'], retrievedTargetLock, 'NA', 'NA', - acctok['endpoint'], relTarget, { + respmd = { 'message': 'Target file already exists', # specs (the WOPI validator) require these to be populated with valid values 'Name': os.path.basename(relTarget), 'Url': utils.generateWopiSrc(statInfo['inode'], acctok['appname'] == srv.proxiedappname), - }) + } + return utils.makeConflictResponse('PUT_RELATIVE', acctok['userid'], retrievedTargetLock, 'NA', 'NA', + acctok['endpoint'], relTarget, respmd) except IOError: # optimistically assume we're clear pass @@ -391,6 +392,9 @@ def putRelative(fileid, reqheaders, acctok): newstat = st.statx(acctok['endpoint'], targetName, acctok['userid']) _, newfileid = common.decodeinode(newstat['inode']) except IOError as e: + if str(e) == common.ACCESS_ERROR: + # BAD_REQUEST may seem better but the WOPI validator tests explicitly expect NOT_IMPLEMENTED + return 'Unauthorized to perform PutRelative', http.client.NOT_IMPLEMENTED utils.storeForRecovery(flask.request.get_data(), acctok['username'], targetName, flask.request.args['access_token'][-20:], e) return IO_ERROR, http.client.INTERNAL_SERVER_ERROR diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 69209fdb..dc118293 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -426,9 +426,11 @@ def storeWopiFile(acctok, retrievedlock, xakey, targetname=''): writeerror = None try: st.writefile(acctok['endpoint'], targetname, acctok['userid'], flask.request.get_data(), - (acctok['appname'], encodeLock(retrievedlock))) + (acctok['appname'], encodeLock(retrievedlock))) except IOError as e: - # in case something goes wrong on write, we still want to setxattr but report this error to the caller + if str(e) == common.ACCESS_ERROR: + raise + # something went wrong on write: we still want to setxattr but report this error to the caller writeerror = e # in all cases save the current time for later conflict checking: this is never older than the mtime of the file st.setxattr(acctok['endpoint'], targetname, acctok['userid'], xakey, int(time.time()), encodeLock(retrievedlock)) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 8c004f10..f97e7fab 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -415,6 +415,9 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): if 'file has a valid extended attribute lock' in rc.message: log.warning('msg="Lock mismatch when writing file" filepath="%s"' % filepath) raise IOError(common.LOCK_MISMATCH_ERROR) + if common.ACCESS_ERROR in rc.message: + log.warning('msg="Access denied when writing file" filepath="%s"' % filepath) + raise IOError(common.ACCESS_ERROR) # any other failure is reported as is log.error('msg="Error opening the file for write" filepath="%s" elapsedTimems="%.1f" error="%s"' % (filepath, (tend-tstart)*1000, rc.message.strip('\n'))) From 17285e6903542b74a88169f36d47c576fcd93ee6 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 9 Nov 2022 09:49:56 +0100 Subject: [PATCH 098/325] CI: relaxed maximum complexity --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 004674b5..63688404 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -30,7 +30,7 @@ jobs: # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide, we further relax this - flake8 . --count --exit-zero --max-complexity=15 --max-line-length=130 --statistics + flake8 . --count --exit-zero --max-complexity=30 --max-line-length=130 --statistics - name: Test with pytest run: | pytest From 9f58bab4fc475dd8aa6917848748f96ae59bc93f Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 14 Nov 2022 12:26:09 +0100 Subject: [PATCH 099/325] xroot: log error in case of timeout on proc/user commands --- src/core/xrootiface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index f97e7fab..00075510 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -106,7 +106,7 @@ def _xrootcmd(endpoint, cmd, subcmd, userid, args): rc, _ = f.open(url, OpenFlags.READ, timeout=timeout) tend = time.time() if not f.is_open(): - log.error('msg="Timeout with xroot" cmd="%s" subcmd="%s" args="%s"' % (cmd, subcmd, args)) + log.error('msg="Error or timeout with xroot" cmd="%s" subcmd="%s" args="%s" rc="%s"' % (cmd, subcmd, args, rc)) raise IOError('Timeout executing %s' % cmd) res = b''.join(f.readlines()).decode().split('&') if len(res) == 3: # we may only just get stdout: in that case, assume it's all OK From 85c0af0ad3e25c51beb09c2ab3d97385fcf0a8a2 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 14 Nov 2022 12:27:22 +0100 Subject: [PATCH 100/325] Explained how to enforce a timeout in xrootd --- wopiserver.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wopiserver.conf b/wopiserver.conf index 90165800..79d479d7 100644 --- a/wopiserver.conf +++ b/wopiserver.conf @@ -181,6 +181,9 @@ chunksize = 4194304 #storagehomepath = /your/top/storage/path # Optional timeout value [seconds] applied to all xroot requests. +# Note that for such value to be enforced you also need to override +# the timeout resolution time (15 [seconds] by default) by setting +# the XRD_TIMEOUTRESOLUTION environment variable. #timeout = 10 From 43288cecbd535c12fa6c64d03f2fdc6ba0294831 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 14 Nov 2022 12:50:31 +0100 Subject: [PATCH 101/325] Log the file version on Lock and PutFile --- src/core/wopi.py | 4 ++-- src/core/wopiutils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index b6bbd6a8..a3219429 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -554,9 +554,9 @@ def putFile(fileid, acctok): # Also, note we can't get a time resolution better than one second! # Anyhow, the EFSS should support versioning for such cases. utils.storeWopiFile(acctok, retrievedLock, utils.LASTSAVETIMEKEY) - log.info('msg="File stored successfully" action="edit" user="%s" filename="%s" token="%s"' % - (acctok['userid'][-20:], acctok['filename'], flask.request.args['access_token'][-20:])) statInfo = st.statx(acctok['endpoint'], acctok['filename'], acctok['userid'], versioninv=1) + log.info('msg="File stored successfully" action="edit" user="%s" filename="%s" version="%s" token="%s"' % + (acctok['userid'][-20:], acctok['filename'], statInfo['etag'], flask.request.args['access_token'][-20:])) resp = flask.Response() resp.status_code = http.client.OK resp.headers['X-WOPI-ItemVersion'] = 'v%s' % statInfo['etag'] diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index dc118293..1ba70ee2 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -405,8 +405,8 @@ def makeLockSuccessResponse(operation, acctok, lock, version): session = acctok['username'] _resolveSession(session, acctok['filename']) - log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s"' % - (operation.title(), acctok['filename'], flask.request.args['access_token'][-20:], session, lock)) + log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s" version="%s"' % + (operation.title(), acctok['filename'], flask.request.args['access_token'][-20:], session, lock, version)) resp = flask.Response() resp.status_code = http.client.OK resp.headers['X-WOPI-ItemVersion'] = version From 05fe010772a829c4c02e16c238c06216e53a0aa5 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 14 Nov 2022 12:37:35 +0100 Subject: [PATCH 102/325] Preparing release --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed8db797..f9509896 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ ## Changelog for the WOPI server +### Mon Nov 14 2022 - v9.3.0 +- Introduced heuristic to log which sessions are allowed + from opening a collaborative session and which ones are + prevented by the application +- Introduced support for app-aware locks in EOS (#94) +- Disabled SaveAs action when user is not owner +- Improved error coverage in case of transient errors + in bridged apps and in PutFile operations +- Moved from LGTM to CodeQL workflow on GitHub (#100) + ### Mon Oct 17 2022 - v9.2.0 - Added option to use file or stream handler for logging (#91) - Introduced configurable hostURLs for CheckFileInfo (#93) From ce34f98d66dd58c1adda61f68ed929d055bf9744 Mon Sep 17 00:00:00 2001 From: Snyk bot Date: Mon, 14 Nov 2022 16:14:31 +0200 Subject: [PATCH 103/325] fix: wopiserver.Dockerfile to reduce vulnerabilities (#98) The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-ALPINE316-EXPAT-3062883 --- wopiserver.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wopiserver.Dockerfile b/wopiserver.Dockerfile index e22f4b47..d31d5004 100644 --- a/wopiserver.Dockerfile +++ b/wopiserver.Dockerfile @@ -2,7 +2,7 @@ # # Build: make docker or docker-compose -f wopiserver.yaml build --build-arg VERSION=`git describe | sed 's/^v//'` wopiserver -FROM python:3.10-alpine +FROM python:3.12.0a1-alpine ARG VERSION=latest From 98c6a16e5cce0597df9c5a21e38347f534b83693 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 15 Nov 2022 14:56:59 +0100 Subject: [PATCH 104/325] Added test for locking open files This test exercises a race condition and is enabled for xroot only for the time being --- src/core/xrootiface.py | 1 + test/test_storageiface.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 00075510..cbaa9d50 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -427,6 +427,7 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): log.error('msg="Error writing the file" filepath="%s" elapsedTimems="%.1f" error="%s"' % (filepath, (tend-tstart)*1000, rc.message.strip('\n'))) raise IOError(rc.message.strip('\n')) + log.debug('msg="Write completed" filepath="%s"' % filepath) rc, _ = f.truncate(size) if not rc.ok: log.error('msg="Error truncating the file" filepath="%s" elapsedTimems="%.1f" error="%s"' % diff --git a/test/test_storageiface.py b/test/test_storageiface.py index ba76fdc0..d3b19d7c 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -26,6 +26,7 @@ class TestStorage(unittest.TestCase): '''Simple tests for the storage layers of the WOPI server. See README for how to run the tests for each storage provider''' initialized = False + storagetype = None @classmethod def globalinit(cls): @@ -45,6 +46,7 @@ def globalinit(cls): storagetype = config.get('general', 'storagetype') cls.userid = config.get(storagetype, 'userid') cls.endpoint = config.get(storagetype, 'endpoint') + cls.storagetype = storagetype except (KeyError, configparser.NoOptionError): print("Missing option or missing configuration, check the wopiserver-test.conf file") raise @@ -333,6 +335,24 @@ def test_expired_locks(self): self.assertIn('File was not locked', str(context.exception)) self.storage.removefile(self.endpoint, self.homepath + '/testelock', self.userid) + def test_lock_open_file(self): + '''Opens a file for write and tries to grab a lock in a race, which shall fail if the file is still open''' + if TestStorage.storagetype != 'xroot': + # Possibly to be implemented in CS3; local interface would always fail this test + raise unittest.SkipTest('Missing feature in %s' % TestStorage.storagetype) + t1 = Thread(target=self.storage.writefile, + args=[self.endpoint, self.homepath + '/testlockopen', self.userid, databuf, None], kwargs={'islock': True}) + t2 = Thread(target=self.storage.setlock, + args=[self.endpoint, self.homepath + '/testlockopen', self.userid, 'test app', 'testlock']) + t1.start() + time.sleep(0.001) + t2.start() + t1.join() + t2.join() + lock = self.storage.getlock(self.endpoint, self.homepath + '/testlockopen', self.userid) + self.storage.removefile(self.endpoint, self.homepath + '/testlockopen', self.userid) + self.assertIsNone(lock, 'Lock was set') + def test_remove_nofile(self): '''Test removal of a non-existing file''' with self.assertRaises(IOError) as context: From a0044e38fc691ae269ade28a9a8d917ee8e02af6 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 18 Nov 2022 16:29:39 +0100 Subject: [PATCH 105/325] fix: requirements.txt to reduce vulnerabilities (#104) The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-SETUPTOOLS-3113904 Co-authored-by: snyk-bot --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 0ef8ce47..50a91b57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ prometheus-flask-exporter cs3apis>=0.1.dev101 waitress wheel>=0.38.0 # not directly required, pinned by Snyk to avoid a vulnerability +setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability From 0b91ac36dfbec31c3fb6f6f6a44c11e96cc39497 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 15 Nov 2022 11:45:26 +0100 Subject: [PATCH 106/325] Implemented PutUserInfo WOPI action, useful for the business flow --- src/core/wopi.py | 18 +++++++++++++++++- src/core/wopiutils.py | 3 +++ src/wopiserver.py | 5 +++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index a3219429..48e9f618 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -89,7 +89,10 @@ def checkFileInfo(fileid, acctok): fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE or notOwner fmd['SupportsRename'] = fmd['UserCanRename'] = enablerename and (acctok['viewmode'] == utils.ViewMode.READ_WRITE) fmd['SupportsContainers'] = False # TODO this is all to be implemented - fmd['SupportsUserInfo'] = False # TODO https://docs.microsoft.com/en-us/openspecs/office_protocols/ms-wopi/371e25ae-e45b-47ab-aec3-9111e962919d + fmd['SupportsUserInfo'] = True + uinfo = st.getxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], utils.USERINFOKEY) + if uinfo: + fmd['UserInfo'] = uinfo # populate app-specific metadata # the following is to enable the 'Edit in Word/Excel/PowerPoint' (desktop) action (probably broken) @@ -572,3 +575,16 @@ def putFile(fileid, acctok): log.warning('msg="Forcing conflict based on save time" user="%s" filename="%s" savetime="%s" lastmtime="%s" token="%s"' % (acctok['userid'][-20:], acctok['filename'], savetime, mtime, flask.request.args['access_token'][-20:])) return utils.storeAfterConflict(acctok, 'External', lock, 'The file being edited got moved or overwritten') + + +def putUserInfo(_fileid, reqbody, acctok): + '''Implements the PutUserInfo WOPI call''' + try: + statInfo = st.statx(acctok['endpoint'], acctok['filename'], acctok['userid']) + lock = st.getlock(acctok['endpoint'], acctok['filename'], acctok['userid']) + st.setxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], utils.USERINFOKEY, reqbody.decode(), + utils.encodeLock(lock) if lock else None) + return 'OK', http.client.OK + except IOError as e: + log.error('msg="PutUserInfo" token="%s" error="%s"' % (flask.request.args['access_token'][-20:], e)) + return IO_ERROR, http.client.INTERNAL_SERVER_ERROR diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 1ba70ee2..15efa802 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -27,6 +27,9 @@ # this is the xattr key used for conflicts resolution on the remote storage LASTSAVETIMEKEY = 'iop.wopi.lastwritetime' +# this is the xattr key used to store user info data from WOPI apps +USERINFOKEY = 'iop.wopi.userinfo' + # header used by reverse proxies such as traefik to pass the real remote IP address REALIPHEADER = 'X-Real-IP' diff --git a/src/wopiserver.py b/src/wopiserver.py index 956c52b5..b58284ea 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -471,7 +471,7 @@ def wopiFilesPost(fileid): acctokOrMsg, httpcode = utils.validateAndLogHeaders(op) if httpcode: return acctokOrMsg, httpcode - if op != 'GET_LOCK' and utils.ViewMode(acctokOrMsg['viewmode']) != utils.ViewMode.READ_WRITE: + if op not in ('GET_LOCK', 'PUT_USER_INFO') and utils.ViewMode(acctokOrMsg['viewmode']) != utils.ViewMode.READ_WRITE: # protect this call if the WOPI client does not have privileges return 'Attempting to perform a write operation using a read-only token', http.client.UNAUTHORIZED if op in ('LOCK', 'REFRESH_LOCK'): @@ -486,7 +486,8 @@ def wopiFilesPost(fileid): return core.wopi.deleteFile(fileid, headers, acctokOrMsg) if op == 'RENAME_FILE': return core.wopi.renameFile(fileid, headers, acctokOrMsg) - # elif op == 'PUT_USER_INFO': + if op == 'PUT_USER_INFO': + return core.wopi.putUserInfo(fileid, flask.request.get_data(), acctokOrMsg) # Any other op is unsupported Wopi.log.warning('msg="Unknown/unsupported operation" operation="%s"' % op) return 'Not supported operation found in header', http.client.NOT_IMPLEMENTED From 2af29c1d77aeeeb277835e688f5ff84dd4dccd4d Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 21 Nov 2022 10:08:51 +0100 Subject: [PATCH 107/325] Further release preparations. Tagging postponed until CI is back --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9509896..afd176c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Changelog for the WOPI server -### Mon Nov 14 2022 - v9.3.0 +### ... Nov .. 2022 - v9.3.0 - Introduced heuristic to log which sessions are allowed from opening a collaborative session and which ones are prevented by the application @@ -9,6 +9,7 @@ - Improved error coverage in case of transient errors in bridged apps and in PutFile operations - Moved from LGTM to CodeQL workflow on GitHub (#100) +- Introduced support for PutUserInfo ### Mon Oct 17 2022 - v9.2.0 - Added option to use file or stream handler for logging (#91) From ead50b5d22890f8748819b16005976593053bb5a Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 21 Nov 2022 18:23:03 +0100 Subject: [PATCH 108/325] Trivial refactoring. Interestingly, this workaround is still needed --- src/core/wopiutils.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 15efa802..5d0669d4 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -344,20 +344,17 @@ def compareWopiLocks(lock1, lock2): log.debug('msg="compareLocks" lock1="%s" lock2="%s" strict="False" result="%r"' % (lock1, lock2, l1['S'] == l2['S'])) return l1['S'] == l2['S'] # used by Word - log.debug('msg="compareLocks" lock1="%s" lock2="%s" strict="False" result="False"' % (lock1, lock2)) - return False except (TypeError, ValueError): # lock2 is not a JSON dictionary if 'S' in l1: log.debug('msg="compareLocks" lock1="%s" lock2="%s" strict="False" result="%r"' % (lock1, lock2, l1['S'] == lock2)) - return l1['S'] == lock2 # also used by Word (BUG!) - log.debug('msg="compareLocks" lock1="%s" lock2="%s" strict="False" result="False"' % (lock1, lock2)) - return False + return l1['S'] == lock2 # also used by Word except (TypeError, ValueError): # lock1 is not a JSON dictionary: log the lock values and fail the comparison - log.debug('msg="compareLocks" lock1="%s" lock2="%s" strict="False" result="False"' % (lock1, lock2)) - return False + pass + log.debug('msg="compareLocks" lock1="%s" lock2="%s" strict="False" result="False"' % (lock1, lock2)) + return False def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, endpoint, filename, reason=None): From f2f39bc7d9ad02fa9479a3c7ec454cb824ba3423 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 22 Nov 2022 09:18:24 +0100 Subject: [PATCH 109/325] Fixed build by explicitly installing g++ --- wopiserver.Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wopiserver.Dockerfile b/wopiserver.Dockerfile index d31d5004..eab5d8cc 100644 --- a/wopiserver.Dockerfile +++ b/wopiserver.Dockerfile @@ -10,10 +10,11 @@ LABEL maintainer="cernbox-admins@cern.ch" \ org.opencontainers.image.title="The ScienceMesh IOP WOPI server" \ org.opencontainers.image.version="$VERSION" -# prerequisites +# prerequisites: we explicitly install g++ as it is required by grpcio but missing from its dependencies WORKDIR /app COPY requirements.txt . -RUN pip3 install --upgrade pip setuptools && \ +RUN apk add g++ && \ + pip3 install --upgrade pip setuptools && \ pip3 install --no-cache-dir --upgrade -r requirements.txt # install software From 7375cb8537b884528cf8a42ed735684ec38f9cf3 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 15 Nov 2022 08:48:08 +0100 Subject: [PATCH 110/325] Added configuration flag to enable the business flow --- src/core/wopi.py | 5 ++++- src/wopiserver.py | 3 +++ wopiserver.conf | 14 +++++++++----- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 48e9f618..e5e56af8 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -71,12 +71,15 @@ def checkFileInfo(fileid, acctok): and srv.config.get('general', 'downloadurl', fallback=None): fmd['DownloadUrl'] = fmd['FileUrl'] = '%s?access_token=%s' % \ (srv.config.get('general', 'downloadurl'), flask.request.args['access_token']) + if srv.config.get('general', 'businessflow', fallback='False').upper() == 'TRUE': + # enable the check for real users, not for public links + fmd['LicenseCheckForEditIsEnabled'] = not fmd['IsAnonymousUser'] fmd['BreadcrumbBrandName'] = srv.config.get('general', 'brandingname', fallback=None) fmd['BreadcrumbBrandUrl'] = srv.config.get('general', 'brandingurl', fallback=None) fmd['OwnerId'] = statInfo['ownerid'] fmd['UserId'] = acctok['wopiuser'] # typically same as OwnerId; different when accessing shared documents fmd['Size'] = statInfo['size'] - # note that in ownCloud the version is generated as: `'V' + etag + checksum` + # note that in ownCloud 10 the version is generated as: `'V' + etag + checksum` fmd['Version'] = 'v%s' % statInfo['etag'] fmd['SupportsExtendedLockLength'] = fmd['SupportsGetLock'] = True fmd['SupportsUpdate'] = fmd['UserCanWrite'] = fmd['SupportsLocks'] = \ diff --git a/src/wopiserver.py b/src/wopiserver.py index b58284ea..6367025e 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -354,6 +354,9 @@ def iopOpenInApp(): res['app-url'] = appurl if vm == utils.ViewMode.READ_WRITE else appviewurl res['app-url'] += '%sWOPISrc=%s' % ('&' if '?' in res['app-url'] else '?', utils.generateWopiSrc(inode, appname == Wopi.proxiedappname)) + if Wopi.config.get('general', 'businessflow', fallback='False').upper() == 'TRUE': + # tells the app to enable the business flow if appropriate + res['app-url'] += '&IsLicensedUser=1' res['form-parameters'] = {'access_token': acctok} Wopi.log.info('msg="iopOpenInApp: redirecting client" appurl="%s"' % res['app-url']) diff --git a/wopiserver.conf b/wopiserver.conf index 79d479d7..e04f2287 100644 --- a/wopiserver.conf +++ b/wopiserver.conf @@ -8,11 +8,15 @@ [general] # Storage access layer to be loaded in order to operate this WOPI server # Supported values: local, xroot, cs3. -#storagetype = xroot +#storagetype = # Port where to listen for WOPI requests port = 8880 +# The internal server engine to use (defaults to flask). +# Set to waitress for production installations. +#internalserver = flask + # Logging level. Debug enables the Flask debug mode as well. # Valid values are: Debug, Info, Warning, Error. loglevel = Info @@ -65,10 +69,6 @@ loghandler = file # a 'Edit in Desktop client' action on Windows-based clients #webdavurl = https://your-efss-server.org/webdav -# The internal server engine to use (defaults to flask). -# Set to waitress for production installations. -#internalserver = flask - # List of file extensions deemed incompatible with LibreOffice: # interoperable locking will be disabled for such files nonofficetypes = .md .zmd .txt @@ -122,6 +122,10 @@ wopilockexpiration = 1800 #wopiproxysecretfile = /path/to/your/shared-key-file #proxiedappname = Name of your proxied app +# A flag to enable the business flow with Microsoft Office as detailed in: +# https://learn.microsoft.com/en-us/microsoft-365/cloud-storage-partner-program/online/scenarios/business +#businessflow = False + ### The following options are deprecated and not to be used with Reva # URL of your Microsoft Office Online service (either local or remote) From 3673749597e218560f7953b8753d9d7bac5f6957 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 22 Nov 2022 11:30:35 +0100 Subject: [PATCH 111/325] Really fixed the build by downgrading to the latest stable alpine docker image --- wopiserver.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wopiserver.Dockerfile b/wopiserver.Dockerfile index eab5d8cc..631541d0 100644 --- a/wopiserver.Dockerfile +++ b/wopiserver.Dockerfile @@ -2,7 +2,7 @@ # # Build: make docker or docker-compose -f wopiserver.yaml build --build-arg VERSION=`git describe | sed 's/^v//'` wopiserver -FROM python:3.12.0a1-alpine +FROM python:3.11-alpine ARG VERSION=latest From 5343953d7b8b7c8e2499ee4df91911c11babe6f3 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 22 Nov 2022 12:03:31 +0100 Subject: [PATCH 112/325] Create SECURITY.md Created minimal security policy for GitHub --- SECURITY.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..fd854407 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,15 @@ +# Security Policy + +## Supported Versions + +By default, only the latest tagged version is supported. + +In case of major issues upgrading to the latest tag, a backport +to a previous release from the same major version can be considered. + +## Reporting a Vulnerability + +Please open a standard issue and mention `Vulnerability:` in the title. + +Depending on the severity, it will be reviewed as part of the +next development cycle. From 308018d29aff964902fcb9535ce6bfff69db8910 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 24 Nov 2022 14:05:53 +0100 Subject: [PATCH 113/325] Removed racy test as it turns out to be too unstable --- test/test_storageiface.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/test_storageiface.py b/test/test_storageiface.py index d3b19d7c..55652ffb 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -335,24 +335,6 @@ def test_expired_locks(self): self.assertIn('File was not locked', str(context.exception)) self.storage.removefile(self.endpoint, self.homepath + '/testelock', self.userid) - def test_lock_open_file(self): - '''Opens a file for write and tries to grab a lock in a race, which shall fail if the file is still open''' - if TestStorage.storagetype != 'xroot': - # Possibly to be implemented in CS3; local interface would always fail this test - raise unittest.SkipTest('Missing feature in %s' % TestStorage.storagetype) - t1 = Thread(target=self.storage.writefile, - args=[self.endpoint, self.homepath + '/testlockopen', self.userid, databuf, None], kwargs={'islock': True}) - t2 = Thread(target=self.storage.setlock, - args=[self.endpoint, self.homepath + '/testlockopen', self.userid, 'test app', 'testlock']) - t1.start() - time.sleep(0.001) - t2.start() - t1.join() - t2.join() - lock = self.storage.getlock(self.endpoint, self.homepath + '/testlockopen', self.userid) - self.storage.removefile(self.endpoint, self.homepath + '/testlockopen', self.userid) - self.assertIsNone(lock, 'Lock was set') - def test_remove_nofile(self): '''Test removal of a non-existing file''' with self.assertRaises(IOError) as context: From 3fc4d2c08536b382e7b5601f23a0d8e7edc6920c Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 24 Nov 2022 14:17:58 +0100 Subject: [PATCH 114/325] Adapted remaining storage calls to use `lockmd` instead of `lockid` `lockmd` is either `None` or a `(appname, lockid)` tuple. This prepares the support for native locking in EOS, as that is based on the `appname` rather than the `lockid`. Also adapted tests and some error messages. --- src/core/commoniface.py | 23 ++---------- src/core/cs3iface.py | 18 +++++++--- src/core/localiface.py | 43 ++++++++++++++++------ src/core/wopi.py | 24 ++++++------- src/core/wopiutils.py | 3 +- src/core/xrootiface.py | 76 +++++++++++++++++++++------------------ test/test_storageiface.py | 21 +++++------ 7 files changed, 114 insertions(+), 94 deletions(-) diff --git a/src/core/commoniface.py b/src/core/commoniface.py index 8f80ea5e..f67c6168 100644 --- a/src/core/commoniface.py +++ b/src/core/commoniface.py @@ -17,10 +17,8 @@ ENOENT_MSG = 'No such file or directory' # standard error thrown when attempting to overwrite a file/xattr in O_EXCL mode -EXCL_ERROR = 'File/xattr exists but EXCL mode requested' - -# error thrown on refreshlock when the payload does not match -LOCK_MISMATCH_ERROR = 'Existing lock payload does not match' +# or when a lock operation cannot be performed because of failed preconditions +EXCL_ERROR = 'File/xattr exists but EXCL mode requested, lock mismatch or lock expired' # standard error thrown when attempting an operation without the required access rights ACCESS_ERROR = 'Operation not permitted' @@ -90,20 +88,3 @@ def decodeinode(inode): '''Decodes an inode obtained from encodeinode()''' e, f = inode.split('!') return e, urlsafe_b64decode(f.encode()).decode() - - -def validatelock(filepath, currlock, appname, value, op, log): - '''Common logic for validating locks in the xrootd and local storage interfaces. - Duplicates some logic implemented in Reva for the cs3 storage interface''' - try: - if not currlock: - raise IOError('File was not locked or lock had expired') - if appname and currlock['app_name'] != appname \ - and currlock['app_name'] != 'wopi' and appname != 'wopi': # TODO deprecated, to be removed after CERNBox rollout - raise IOError('File is locked by %s' % currlock['app_name']) - if value != currlock['lock_id']: - raise IOError(LOCK_MISMATCH_ERROR) - except IOError as e: - log.warning('msg="Failed to %s" filepath="%s" appname="%s" lockid="%s" currlock="%s" reason="%s"' % - (op, filepath, appname, value, currlock, e)) - raise diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index 78535ef3..23d1005b 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -123,11 +123,14 @@ def statx(endpoint, fileref, userid, versioninv=1): return stat(endpoint, fileref, userid, versioninv) -def setxattr(endpoint, filepath, userid, key, value, lockid): +def setxattr(endpoint, filepath, userid, key, value, lockmd): '''Set the extended attribute to using the given userid as access token''' reference = _getcs3reference(endpoint, filepath) md = cs3spr.ArbitraryMetadata() md.metadata.update({key: str(value)}) # pylint: disable=no-member + lockid = None + if lockmd: + _, lockid = lockmd req = cs3sp.SetArbitraryMetadataRequest(ref=reference, arbitrary_metadata=md, lock_id=lockid) res = ctx['cs3gw'].SetArbitraryMetadata(request=req, metadata=[('x-access-token', userid)]) if res.status.code != cs3code.CODE_OK: @@ -162,9 +165,12 @@ def getxattr(endpoint, filepath, userid, key): return None -def rmxattr(endpoint, filepath, userid, key, lockid): +def rmxattr(endpoint, filepath, userid, key, lockmd): '''Remove the extended attribute using the given userid as access token''' reference = _getcs3reference(endpoint, filepath) + lockid = None + if lockmd: + _, lockid = lockmd req = cs3sp.UnsetArbitraryMetadataRequest(ref=reference, arbitrary_metadata_keys=[key], lock_id=lockid) res = ctx['cs3gw'].UnsetArbitraryMetadata(request=req, metadata=[('x-access-token', userid)]) if res.status.code != cs3code.CODE_OK: @@ -318,7 +324,7 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): log.error('msg="Failed to initiateFileUpload on write" filepath="%s" trace="%s" code="%s" reason="%s"' % (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) if '_lock_' in res.status.message: # TODO find the error code returned by Reva once this is implemented - raise IOError(common.LOCK_MISMATCH_ERROR) + raise IOError(common.EXCL_ERROR) raise IOError(res.status.message) tend = time.time() log.debug('msg="writefile: InitiateFileUploadRes returned" trace="%s" protocols="%s"' % @@ -346,11 +352,13 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): (filepath, (tend - tstart) * 1000, islock)) -def renamefile(endpoint, filepath, newfilepath, userid, lockid): +def renamefile(endpoint, filepath, newfilepath, userid, lockmd): '''Rename a file from origfilepath to newfilepath using the given userid as access token.''' reference = _getcs3reference(endpoint, filepath) newfileref = _getcs3reference(endpoint, newfilepath) - + lockid = None + if lockmd: + _, lockid = lockmd req = cs3sp.MoveRequest(source=reference, destination=newfileref, lock_id=lockid) res = ctx['cs3gw'].Move(request=req, metadata=[('x-access-token', userid)]) if res.status.code in [cs3code.CODE_FAILED_PRECONDITION, cs3code.CODE_ABORTED]: diff --git a/src/core/localiface.py b/src/core/localiface.py index 8e54701e..0eefacbd 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -102,13 +102,33 @@ def statx(endpoint, filepath, userid, versioninv=1): return stat(endpoint, filepath, userid) -def setxattr(endpoint, filepath, userid, key, value, lockid): +def _validatelock(filepath, currlock, lockmd, op, log): + '''Common logic for validating locks: duplicates some logic + natively implemented by EOS and Reva on the other storage interfaces''' + appname = value = None + if lockmd: + appname, value = lockmd + try: + if not currlock: + raise IOError(common.EXCL_ERROR) + if appname and currlock['app_name'] != appname \ + and currlock['app_name'] != 'wopi' and appname != 'wopi': # TODO deprecated, to be removed after CERNBox rollout + raise IOError(common.EXCL_ERROR + ', file is locked by %s' % currlock['app_name']) + if value != currlock['lock_id']: + raise IOError(common.EXCL_ERROR) + except IOError as e: + log.warning('msg="Failed to %s" filepath="%s" appname="%s" lockid="%s" currlock="%s" reason="%s"' % + (op, filepath, appname, value, currlock, e)) + raise + + +def setxattr(endpoint, filepath, userid, key, value, lockmd): '''Set the extended attribute to on behalf of the given userid''' if key != common.LOCKKEY: currlock = getlock(endpoint, filepath, userid) if currlock: # enforce lock only if previously set - common.validatelock(filepath, currlock, None, lockid, 'setxattr', log) + _validatelock(filepath, currlock, lockmd, 'setxattr', log) try: os.setxattr(_getfilepath(filepath), 'user.' + key, str(value).encode()) except OSError as e: @@ -125,10 +145,10 @@ def getxattr(_endpoint, filepath, _userid, key): return None -def rmxattr(endpoint, filepath, userid, key, lockid): +def rmxattr(endpoint, filepath, userid, key, lockmd): '''Remove the extended attribute on behalf of the given userid''' if key != common.LOCKKEY: - common.validatelock(filepath, getlock(endpoint, filepath, userid), None, lockid, 'rmxattr', log) + _validatelock(filepath, getlock(endpoint, filepath, userid), lockmd, 'rmxattr', log) try: os.removexattr(_getfilepath(filepath), 'user.' + key) except OSError as e: @@ -173,7 +193,7 @@ def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): if not oldvalue and currlock: # this is a pure refresh operation oldvalue = currlock['lock_id'] - common.validatelock(filepath, currlock, appname, oldvalue, 'refreshlock', log) + _validatelock(filepath, currlock, (appname, oldvalue), 'refreshlock', log) # this is non-atomic, but if we get here the lock was already held log.debug('msg="Invoked refreshlock" filepath="%s" value="%s"' % (filepath, value)) setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value), None) @@ -181,7 +201,7 @@ def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): def unlock(endpoint, filepath, userid, appname, value): '''Remove the lock as an xattr on behalf of the given userid''' - common.validatelock(filepath, getlock(endpoint, filepath, userid), appname, value, 'unlock', log) + _validatelock(filepath, getlock(endpoint, filepath, userid), (appname, value), 'unlock', log) log.debug('msg="Invoked unlock" filepath="%s" value="%s"' % (filepath, value)) rmxattr(endpoint, filepath, '0:0', common.LOCKKEY, None) @@ -217,10 +237,9 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): content = bytes(content, 'UTF-8') size = len(content) if lockmd: - appname, lockid = lockmd - common.validatelock(filepath, getlock(endpoint, filepath, userid), appname, lockid, 'writefile', log) + _validatelock(filepath, getlock(endpoint, filepath, userid), lockmd, 'writefile', log) elif getlock(endpoint, filepath, userid): - raise IOError(common.LOCK_MISMATCH_ERROR) + raise IOError(common.EXCL_ERROR) log.debug('msg="Invoking writeFile" filepath="%s" size="%d"' % (filepath, size)) tstart = time.time() if islock: @@ -255,8 +274,12 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): (filepath, (tend - tstart) * 1000, islock)) -def renamefile(endpoint, origfilepath, newfilepath, _userid, _lockid): +def renamefile(endpoint, origfilepath, newfilepath, userid, lockmd): '''Rename a file from origfilepath to newfilepath on behalf of the given userid.''' + currlock = getlock(endpoint, origfilepath, userid) + if currlock: + # enforce lock only if previously set + _validatelock(origfilepath, currlock, lockmd, 'renamefile', log) try: os.rename(_getfilepath(origfilepath), _getfilepath(newfilepath)) except OSError as e: diff --git a/src/core/wopi.py b/src/core/wopi.py index e5e56af8..f826902a 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -244,8 +244,8 @@ def setLock(fileid, reqheaders, acctok): # on first lock, set an xattr with the current time for later conflicts checking try: - st.setxattr(acctok['endpoint'], fn, acctok['userid'], utils.LASTSAVETIMEKEY, - int(time.time()), utils.encodeLock(lock)) + st.setxattr(acctok['endpoint'], fn, acctok['userid'], utils.LASTSAVETIMEKEY, int(time.time()), + (acctok['appname'], utils.encodeLock(lock))) except IOError as e: # not fatal, but will generate a conflict file later on, so log a warning log.warning('msg="Unable to set lastwritetime xattr" lockop="%s" user="%s" filename="%s" token="%s" reason="%s"' % @@ -262,10 +262,10 @@ def setLock(fileid, reqheaders, acctok): # validate against either the given lock (RefreshLock case) or the given old lock (UnlockAndRelock case) if retrievedLock and not utils.compareWopiLocks(retrievedLock, (oldLock if oldLock else lock)): # lock mismatch, the WOPI client is supposed to acknowledge the existing lock - # or deny write access to the file + # and deny access to the file in edit mode otherwise return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, acctok['endpoint'], fn, 'The file is locked by %s' % - (lockHolder if lockHolder != 'wopi' else 'another online editor')) + (lockHolder if lockHolder != 'wopi' else 'another online editor')) # TODO cleanup 'wopi' case # else it's our own lock, refresh it (rechecking the oldLock if necessary, for atomicity) and return try: @@ -298,7 +298,7 @@ def unlock(fileid, reqheaders, acctok): retrievedLock, _ = utils.retrieveWopiLock(fileid, 'UNLOCK', lock, acctok) if not utils.compareWopiLocks(retrievedLock, lock): return utils.makeConflictResponse('UNLOCK', acctok['userid'], retrievedLock, lock, 'NA', - acctok['endpoint'], acctok['filename'], 'Lock mismatch') + acctok['endpoint'], acctok['filename'], 'Lock mismatch unlocking file') # OK, the lock matches, remove it try: # validate that the underlying file is still there @@ -471,7 +471,7 @@ def renameFile(fileid, reqheaders, acctok): retrievedLock, _ = utils.retrieveWopiLock(fileid, 'RENAMEFILE', lock, acctok) if retrievedLock is not None and not utils.compareWopiLocks(retrievedLock, lock): return utils.makeConflictResponse('RENAMEFILE', acctok['userid'], retrievedLock, lock, 'NA', - acctok['endpoint'], acctok['filename']) + acctok['endpoint'], acctok['filename'], 'Lock mismatch renaming file') try: # the destination name comes without base path and typically without extension targetName = os.path.dirname(acctok['filename']) + os.path.sep + targetName \ @@ -482,9 +482,9 @@ def renameFile(fileid, reqheaders, acctok): # try to rename and pass the lock if present. Note that WOPI specs do not require files to be locked # on rename operations, but the backend may still fail as renames may be implemented as copy + delete, # which may require to pass a lock. - st.renamefile(acctok['endpoint'], acctok['filename'], targetName, acctok['userid'], - utils.encodeLock(retrievedLock) if retrievedLock else None) - # also rename the lock if applicable + lockmd = (acctok['appname'], utils.encodeLock(retrievedLock)) if retrievedLock else None + st.renamefile(acctok['endpoint'], acctok['filename'], targetName, acctok['userid'], lockmd) + # also rename the LO lock if applicable if os.path.splitext(acctok['filename'])[1] in srv.codetypes: st.renamefile(acctok['endpoint'], utils.getLibreOfficeLockName(acctok['filename']), utils.getLibreOfficeLockName(targetName), acctok['userid'], None) @@ -584,9 +584,9 @@ def putUserInfo(_fileid, reqbody, acctok): '''Implements the PutUserInfo WOPI call''' try: statInfo = st.statx(acctok['endpoint'], acctok['filename'], acctok['userid']) - lock = st.getlock(acctok['endpoint'], acctok['filename'], acctok['userid']) - st.setxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], utils.USERINFOKEY, reqbody.decode(), - utils.encodeLock(lock) if lock else None) + lockmd = st.getlock(acctok['endpoint'], acctok['filename'], acctok['userid']) + lockmd = (acctok['appname'], utils.encodeLock(lockmd)) if lockmd else None + st.setxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], utils.USERINFOKEY, reqbody.decode(), lockmd) return 'OK', http.client.OK except IOError as e: log.error('msg="PutUserInfo" token="%s" error="%s"' % (flask.request.args['access_token'][-20:], e)) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 5d0669d4..cd19fc32 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -433,7 +433,8 @@ def storeWopiFile(acctok, retrievedlock, xakey, targetname=''): # something went wrong on write: we still want to setxattr but report this error to the caller writeerror = e # in all cases save the current time for later conflict checking: this is never older than the mtime of the file - st.setxattr(acctok['endpoint'], targetname, acctok['userid'], xakey, int(time.time()), encodeLock(retrievedlock)) + st.setxattr(acctok['endpoint'], targetname, acctok['userid'], xakey, int(time.time()), + (acctok['appname'], encodeLock(retrievedlock))) if writeerror: raise writeerror diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index cbaa9d50..6fa23f02 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -74,6 +74,17 @@ def _geturlfor(endpoint): return endpoint if endpoint.find('root://') == 0 else ('root://' + endpoint.replace('newproject', 'eosproject') + '.cern.ch') +def _appforlock(appname): + '''One-liner to generate the app name used for eos locks''' + return 'wopi_' + appname.replace(' ', '_').lower() + + +def _geneoslock(appname): + '''One-liner to generate an EOS app lock. Type is `shared` (hardcoded) for WOPI apps, `exclusive` is also supported''' + return 'expires:%d,type:shared,owner:*:%s' % \ + (int(time.time()) + config.getint("general", "wopilockexpiration"), _appforlock(appname)) + + def _eosargs(userid, app='wopi', bookingsize=0): '''Assume userid is in the form uid:gid and split it into uid, gid plus generate extra EOS-specific arguments for the xroot URL''' @@ -84,23 +95,19 @@ def _eosargs(userid, app='wopi', bookingsize=0): raise ValueError ruid = int(userid[0]) rgid = int(userid[1]) + if app not in ('wopi', 'fuse::wopi'): + app = _appforlock(app) return '?eos.ruid=%d&eos.rgid=%d' % (ruid, rgid) + '&eos.app=' + app + \ (('&eos.bookingsize=' + str(bookingsize)) if bookingsize else '') except (ValueError, IndexError): raise ValueError('Only Unix-based userid is supported with xrootd storage: %s' % userid) -def _geneoslock(appname): - '''One-liner to generate an EOS app lock. Type is `shared` (hardcoded) for WOPI apps, `exclusive` is also supported''' - return 'expires:%d,type:shared,owner:*:wopi_%s' % \ - (int(time.time()) + config.getint("general", "wopilockexpiration"), appname.replace(' ', '_').lower()) - - -def _xrootcmd(endpoint, cmd, subcmd, userid, args): +def _xrootcmd(endpoint, cmd, subcmd, userid, args, app='wopi'): '''Perform the / action on the special /proc/user path on behalf of the given userid. Note that this is entirely EOS-specific.''' with XrdClient.File() as f: - url = _geturlfor(endpoint) + '//proc/user/' + _eosargs(userid) + '&mgm.cmd=' + cmd + \ + url = _geturlfor(endpoint) + '//proc/user/' + _eosargs(userid, app) + '&mgm.cmd=' + cmd + \ ('&mgm.subcmd=' + subcmd if subcmd else '') + '&' + args tstart = time.time() rc, _ = f.open(url, OpenFlags.READ, timeout=timeout) @@ -260,20 +267,17 @@ def statx(endpoint, fileref, userid, versioninv=1): } -def setxattr(endpoint, filepath, userid, key, value, lockid): +def setxattr(endpoint, filepath, _userid, key, value, lockmd): '''Set the extended attribute to via a special open. The userid is overridden to make sure it also works on shared files.''' - if key not in (EOSLOCKKEY, common.LOCKKEY): - currlock = getlock(endpoint, filepath, userid) - if currlock: - # enforce lock only if previously set - common.validatelock(filepath, currlock, None, lockid, 'setxattr', log) - # else skip the check as we're setting the lock itself + appname = 'wopi' + if lockmd: + appname, _ = lockmd if 'user' not in key and 'sys' not in key: # if nothing is given, assume it's a user attr key = 'user.' + key _xrootcmd(endpoint, 'attr', 'set', '0:0', 'mgm.attr.key=' + key + '&mgm.attr.value=' + str(value) - + '&mgm.path=' + _getfilepath(filepath, encodeamp=True)) + + '&mgm.path=' + _getfilepath(filepath, encodeamp=True), appname) def getxattr(endpoint, filepath, _userid, key): @@ -291,15 +295,17 @@ def getxattr(endpoint, filepath, _userid, key): return None -def rmxattr(endpoint, filepath, userid, key, lockid): +def rmxattr(endpoint, filepath, _userid, key, lockmd): '''Remove the extended attribute via a special open. The userid is overridden to make sure it also works on shared files.''' - if key not in (EOSLOCKKEY, common.LOCKKEY): - common.validatelock(filepath, getlock(endpoint, filepath, userid), None, lockid, 'rmxattr', log) + appname = 'wopi' + if lockmd: + appname, _ = lockmd if 'user' not in key and 'sys' not in key: # if nothing is given, assume it's a user attr key = 'user.' + key - _xrootcmd(endpoint, 'attr', 'rm', '0:0', 'mgm.attr.key=' + key + '&mgm.path=' + _getfilepath(filepath, encodeamp=True)) + _xrootcmd(endpoint, 'attr', 'rm', '0:0', + 'mgm.attr.key=' + key + '&mgm.path=' + _getfilepath(filepath, encodeamp=True), appname) def setlock(endpoint, filepath, userid, appname, value, recurse=False): @@ -308,16 +314,15 @@ def setlock(endpoint, filepath, userid, appname, value, recurse=False): try: log.debug('msg="Invoked setlock" filepath="%s" value="%s"' % (filepath, value)) setxattr(endpoint, filepath, userid, EOSLOCKKEY, _geneoslock(appname) + '&mgm.option=c', None) - setxattr(endpoint, filepath, userid, common.LOCKKEY, common.genrevalock(appname, value), None) + setxattr(endpoint, filepath, userid, common.LOCKKEY, common.genrevalock(appname, value), (appname, None)) except IOError as e: - if EXCL_XATTR_MSG in str(e): + if common.EXCL_ERROR in str(e): # check for pre-existing stale locks (this is now not atomic) if not getlock(endpoint, filepath, userid) and not recurse: - rmxattr(endpoint, filepath, userid, EOSLOCKKEY, None) setlock(endpoint, filepath, userid, appname, value, recurse=True) else: - # the lock is valid, raise conflict error - raise IOError(common.EXCL_ERROR) + # the lock is valid + raise else: # we got a different remote error, raise it raise @@ -392,18 +397,16 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): O_CREAT|O_EXCL, preventing race conditions.''' size = len(content) log.debug('msg="Invoking writeFile" filepath="%s" userid="%s" size="%d" islock="%s"' % (filepath, userid, size, islock)) - if lockmd: - appname, _ = lockmd - else: - appname = '' - f = XrdClient.File() - tstart = time.time() if islock: # this is required to trigger the O_EXCL behavior on EOS when creating lock files appname = 'fuse::wopi' - else: + elif lockmd: # this is exclusively used to validate the lock with the app as holder, according to EOS specs (cf. _geneoslock()) - appname = 'wopi_' + appname.replace(' ', '_').lower() + appname, _ = lockmd + else: + appname = 'wopi' + f = XrdClient.File() + tstart = time.time() rc, _ = f.open(_geturlfor(endpoint) + '/' + homepath + filepath + _eosargs(userid, appname, size), OpenFlags.NEW if islock else OpenFlags.DELETE, timeout=timeout) tend = time.time() @@ -442,11 +445,14 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): (filepath, (tend-tstart)*1000, islock)) -def renamefile(endpoint, origfilepath, newfilepath, userid, _lockid): +def renamefile(endpoint, origfilepath, newfilepath, userid, lockmd): '''Rename a file via a special open from origfilepath to newfilepath on behalf of the given userid.''' + appname = 'wopi' + if lockmd: + appname, _ = lockmd _xrootcmd(endpoint, 'file', 'rename', userid, 'mgm.path=' + _getfilepath(origfilepath, encodeamp=True) + '&mgm.file.source=' + _getfilepath(origfilepath, encodeamp=True) - + '&mgm.file.target=' + _getfilepath(newfilepath, encodeamp=True)) + + '&mgm.file.target=' + _getfilepath(newfilepath, encodeamp=True), appname) def removefile(endpoint, filepath, userid, force=False): diff --git a/test/test_storageiface.py b/test/test_storageiface.py index 55652ffb..22cf674f 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -18,7 +18,7 @@ from getpass import getpass sys.path.append('src') # for tests out of the git repo sys.path.append('/app') # for tests within the Docker image -from core.commoniface import EXCL_ERROR, LOCK_MISMATCH_ERROR, ENOENT_MSG # noqa: E402 +from core.commoniface import EXCL_ERROR, ENOENT_MSG # noqa: E402 databuf = b'ebe5tresbsrdthbrdhvdtr' @@ -238,11 +238,11 @@ def test_refresh_lock(self): self.assertIsInstance(statInfo, dict) with self.assertRaises(IOError) as context: self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'test app', 'testlock') - self.assertIn('File was not locked', str(context.exception)) + self.assertEqual(EXCL_ERROR, str(context.exception)) self.storage.setlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'test app', 'testlock') with self.assertRaises(IOError) as context: self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'test app', 'newlock', 'mismatch') - self.assertIn(LOCK_MISMATCH_ERROR, str(context.exception)) + self.assertEqual(EXCL_ERROR, str(context.exception)) self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'test app', 'newlock', 'testlock') l = self.storage.getlock(self.endpoint, self.homepath + '/testrlock', self.userid) # noqa: E741 self.assertIsInstance(l, dict) @@ -253,7 +253,7 @@ def test_refresh_lock(self): self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'test app', 'newlock') with self.assertRaises(IOError) as context: self.storage.refreshlock(self.endpoint, self.homepath + '/testrlock', self.userid, 'mismatched app', 'newlock') - self.assertIn('File is locked by test app', str(context.exception)) + self.assertIn(EXCL_ERROR, str(context.exception)) self.storage.removefile(self.endpoint, self.homepath + '/testrlock', self.userid) def test_lock_race(self): @@ -294,7 +294,8 @@ def test_lock_operations(self): with self.assertRaises(IOError): self.storage.writefile(self.endpoint, self.homepath + '/testlockop', self.userid, databuf, None) with self.assertRaises(IOError): - self.storage.setxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', 123, 'mismatchlock') + self.storage.setxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', 123, + ('mismatch app', 'mismatchlock')) with self.assertRaises(IOError): self.storage.setxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', 123, None) with self.assertRaises(IOError): @@ -302,7 +303,7 @@ def test_lock_operations(self): for chunk in self.storage.readfile(self.endpoint, self.homepath + '/testlockop', self.userid, None): self.assertNotIsInstance(chunk, IOError, 'raised by storage.readfile, lock shall be shared') self.storage.renamefile(self.endpoint, self.homepath + '/testlockop', self.homepath + '/testlockop_renamed', - self.userid, 'testlock') + self.userid, ('test app', 'testlock')) self.storage.removefile(self.endpoint, self.homepath + '/testlockop_renamed', self.userid) def test_expired_locks(self): @@ -327,12 +328,12 @@ def test_expired_locks(self): time.sleep(2.1) with self.assertRaises(IOError) as context: self.storage.refreshlock(self.endpoint, self.homepath + '/testelock', self.userid, 'test app', 'testlock4') - self.assertIn('File was not locked', str(context.exception)) + self.assertEqual(EXCL_ERROR, str(context.exception)) self.storage.setlock(self.endpoint, self.homepath + '/testelock', self.userid, 'test app', 'testlock5') time.sleep(2.1) with self.assertRaises(IOError) as context: self.storage.unlock(self.endpoint, self.homepath + '/testelock', self.userid, 'test app', 'testlock5') - self.assertIn('File was not locked', str(context.exception)) + self.assertEqual(EXCL_ERROR, str(context.exception)) self.storage.removefile(self.endpoint, self.homepath + '/testelock', self.userid) def test_remove_nofile(self): @@ -346,10 +347,10 @@ def test_xattr(self): self.storage.writefile(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, databuf, None) self.storage.setxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 123, None) self.storage.setlock(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'test app', 'xattrlock') - self.storage.setxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 123, 'xattrlock') + self.storage.setxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 123, ('test app', 'xattrlock')) v = self.storage.getxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey') self.assertEqual(v, '123') - self.storage.rmxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 'xattrlock') + self.storage.rmxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', ('test app', 'xattrlock')) v = self.storage.getxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey') self.assertEqual(v, None) self.storage.removefile(self.endpoint, self.homepath + '/test&xattr.txt', self.userid) From e6dbe836ff8f1fd6979b62a5f99db53ec7c41356 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 24 Nov 2022 14:53:26 +0100 Subject: [PATCH 115/325] Added Codacy badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b55d0fb9..9f7c7e88 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Gitter chat](https://badges.gitter.im/cs3org/wopiserver.svg)](https://gitter.im/cs3org/wopiserver) [![Build Status](https://drone.cernbox.cern.ch/api/badges/cs3org/wopiserver/status.svg)](https://drone.cernbox.cern.ch/cs3org/wopiserver) + [![Codacy Badge](https://app.codacy.com/project/badge/Grade/e4e7c46c39b04bddbf63ade4cacdcc7d)](https://www.codacy.com/gh/cs3org/wopiserver/dashboard?utm_source=github.com&utm_medium=referral&utm_content=cs3org/wopiserver&utm_campaign=Badge_Grade) [![codecov](https://codecov.io/gh/cs3org/wopiserver/branch/master/graph/badge.svg)](https://codecov.io/gh/cs3org/wopiserver) ======== From d99d925d8a1bc15de585cf6fdc519a5557ada5d2 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 24 Nov 2022 14:20:34 +0100 Subject: [PATCH 116/325] Complete implementation of EOS locks for EOS v4.8.94+ Also refined lock operations test --- src/core/xrootiface.py | 53 +++++++++++++++++++++++---------------- test/test_storageiface.py | 13 ++++++---- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 6fa23f02..3e790e18 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -16,6 +16,8 @@ EOSVERSIONPREFIX = '.sys.v#.' EXCL_XATTR_MSG = 'exclusive set for existing attribute' +LOCK_MISMATCH_MSG = 'file has a valid extended attribute lock' +FOREIGN_XATTR_MSG = 'foreign attribute lock existing' OK_MSG = '[SUCCESS]' # this is what xroot returns on success EOSLOCKKEY = 'sys.app.lock' @@ -123,13 +125,17 @@ def _xrootcmd(endpoint, cmd, subcmd, userid, args, app='wopi'): # failure: get info from stderr, log and raise msg = res[1][res[1].find('=') + 1:].strip('\n') if common.ENOENT_MSG.lower() in msg or 'unable to get attribute' in msg or rc == '2': - log.info('msg="Invoked xroot on non-existing entity" cmd="%s" subcmd="%s" args="%s" result="%s" rc="%s"' % - (cmd, subcmd, args, msg.replace('error:', ''), rc)) + log.info('msg="Invoked cmd on non-existing entity" cmd="%s" subcmd="%s" args="%s" result="%s" rc="%s"' % + (cmd, subcmd, args, msg.replace('error:', ''), rc.strip('\00'))) raise IOError(common.ENOENT_MSG) if EXCL_XATTR_MSG in msg: log.info('msg="Invoked setxattr on an already locked entity" args="%s" result="%s" rc="%s"' % (args, msg.replace('error:', ''), rc.strip('\00'))) - raise IOError(EXCL_XATTR_MSG) + raise IOError(common.EXCL_ERROR) + if LOCK_MISMATCH_MSG or FOREIGN_XATTR_MSG in msg: + log.info('msg="Mismatched lock" cmd="%s" subcmd="%s" args="%s" app="%s" result="%s" rc="%s"' % + (cmd, subcmd, args, app, msg.replace('error:', ''), rc.strip('\00'))) + raise IOError(common.EXCL_ERROR) # anything else (including permission errors) are logged as errors log.error('msg="Error with xroot" cmd="%s" subcmd="%s" args="%s" error="%s" rc="%s"' % (cmd, subcmd, args, msg, rc.strip('\00'))) @@ -229,7 +235,7 @@ def statx(endpoint, fileref, userid, versioninv=1): tend = time.time() try: if not infov: - raise IOError('xrdquery returned nothing, rcv=%s' + rcv) + raise IOError('xrdquery returned nothing, rcv=%s' % rcv) infov = infov.decode() if OK_MSG not in str(rcv) or 'retc=2' in infov: # the version folder does not exist: create it (on behalf of the owner) as it is done in Reva @@ -267,12 +273,17 @@ def statx(endpoint, fileref, userid, versioninv=1): } -def setxattr(endpoint, filepath, _userid, key, value, lockmd): +def setxattr(endpoint, filepath, userid, key, value, lockmd): '''Set the extended attribute to via a special open. The userid is overridden to make sure it also works on shared files.''' appname = 'wopi' + lockid = None if lockmd: - appname, _ = lockmd + appname, lockid = lockmd + if key not in (EOSLOCKKEY, common.LOCKKEY): + currlock = getlock(endpoint, filepath, userid) + if currlock and currlock['lock_id'] != lockid: + raise IOError(common.EXCL_ERROR) if 'user' not in key and 'sys' not in key: # if nothing is given, assume it's a user attr key = 'user.' + key @@ -337,7 +348,8 @@ def getlock(endpoint, filepath, userid): log.debug('msg="Invoked getlock" filepath="%s"' % filepath) return lock # otherwise, the lock had expired: drop it and return None - log.debug('msg="getlock: removed stale lock" filepath="%s"' % filepath) + log.debug('msg="getlock: removing stale lock" filepath="%s"' % filepath) + rmxattr(endpoint, filepath, userid, EOSLOCKKEY, None) rmxattr(endpoint, filepath, userid, common.LOCKKEY, None) return None @@ -345,22 +357,21 @@ def getlock(endpoint, filepath, userid): def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): '''Refresh the lock value as an xattr''' currlock = getlock(endpoint, filepath, userid) - if not oldvalue and currlock: - # this is a pure refresh operation - oldvalue = currlock['lock_id'] - common.validatelock(filepath, currlock, appname, oldvalue, 'refreshlock', log) + if not currlock or (oldvalue and currlock['lock_id'] != oldvalue): + raise IOError(common.EXCL_ERROR) log.debug('msg="Invoked refreshlock" filepath="%s" value="%s"' % (filepath, value)) # this is non-atomic, but the lock was already held - setxattr(endpoint, filepath, userid, EOSLOCKKEY, _geneoslock(appname), None) - setxattr(endpoint, filepath, userid, common.LOCKKEY, common.genrevalock(appname, value), None) + setxattr(endpoint, filepath, userid, EOSLOCKKEY, _geneoslock(appname), (appname, None)) + setxattr(endpoint, filepath, userid, common.LOCKKEY, common.genrevalock(appname, value), (appname, None)) def unlock(endpoint, filepath, userid, appname, value): '''Remove a lock as an xattr''' - common.validatelock(filepath, getlock(endpoint, filepath, userid), appname, value, 'unlock', log) + if not getlock(endpoint, filepath, userid): + raise IOError(common.EXCL_ERROR) log.debug('msg="Invoked unlock" filepath="%s" value="%s"' % (filepath, value)) - rmxattr(endpoint, filepath, userid, common.LOCKKEY, None) - rmxattr(endpoint, filepath, userid, EOSLOCKKEY, None) + rmxattr(endpoint, filepath, userid, common.LOCKKEY, (appname, None)) + rmxattr(endpoint, filepath, userid, EOSLOCKKEY, (appname, None)) def readfile(endpoint, filepath, userid, _lockid): @@ -377,8 +388,8 @@ def readfile(endpoint, filepath, userid, _lockid): log.info('msg="File not found on read" filepath="%s"' % filepath) yield IOError(common.ENOENT_MSG) else: - log.warning('msg="Error opening the file for read" filepath="%s" code="%d" error="%s"' % - (filepath, rc.shellcode, rc.message.strip('\n'))) + log.error('msg="Error opening the file for read" filepath="%s" code="%d" error="%s"' % + (filepath, rc.shellcode, rc.message.strip('\n'))) yield IOError(rc.message) else: log.info('msg="File open for read" filepath="%s" elapsedTimems="%.1f"' % (filepath, (tend - tstart) * 1000)) @@ -415,9 +426,9 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): # racing against an existing file log.info('msg="File exists on write but islock flag requested" filepath="%s"' % filepath) raise IOError(common.EXCL_ERROR) - if 'file has a valid extended attribute lock' in rc.message: - log.warning('msg="Lock mismatch when writing file" filepath="%s"' % filepath) - raise IOError(common.LOCK_MISMATCH_ERROR) + if LOCK_MISMATCH_MSG in rc.message: + log.warning('msg="Lock mismatch when writing file" app="%s" filepath="%s"' % (appname, filepath)) + raise IOError(common.EXCL_ERROR) if common.ACCESS_ERROR in rc.message: log.warning('msg="Access denied when writing file" filepath="%s"' % filepath) raise IOError(common.ACCESS_ERROR) diff --git a/test/test_storageiface.py b/test/test_storageiface.py index 22cf674f..76037912 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -286,22 +286,25 @@ def test_lock_operations(self): self.storage.setlock(self.endpoint, self.homepath + '/testlockop', self.userid, 'test app', 'testlock') self.storage.writefile(self.endpoint, self.homepath + '/testlockop', self.userid, databuf, ('test app', 'testlock')) with self.assertRaises(IOError): - # TODO different interfaces raise exceptions on either mismatching app xor mismatching lock payload, + # Note that different interfaces raise exceptions on either mismatching app xor mismatching lock payload, # this is why we test that both mismatch. Could be improved, though we specifically care about the lock paylaod. self.storage.writefile(self.endpoint, self.homepath + '/testlockop', self.userid, databuf, - ('mismatchapp', 'mismatchlock')) - self.storage.refreshlock(self.endpoint, self.homepath + '/testlockop', self.userid, 'test app', 'testlock') - with self.assertRaises(IOError): - self.storage.writefile(self.endpoint, self.homepath + '/testlockop', self.userid, databuf, None) + ('mismatch app', 'mismatchlock')) with self.assertRaises(IOError): + # same as above self.storage.setxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', 123, ('mismatch app', 'mismatchlock')) with self.assertRaises(IOError): self.storage.setxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', 123, None) with self.assertRaises(IOError): self.storage.rmxattr(self.endpoint, self.homepath + '/testlockop', self.userid, 'testkey', None) + self.storage.refreshlock(self.endpoint, self.homepath + '/testlockop', self.userid, 'test app', 'testlock') + with self.assertRaises(IOError): + self.storage.refreshlock(self.endpoint, self.homepath + '/testlockop', self.userid, 'mismatched app', 'mismatchlock') for chunk in self.storage.readfile(self.endpoint, self.homepath + '/testlockop', self.userid, None): self.assertNotIsInstance(chunk, IOError, 'raised by storage.readfile, lock shall be shared') + with self.assertRaises(IOError): + self.storage.writefile(self.endpoint, self.homepath + '/testlockop', self.userid, databuf, None) self.storage.renamefile(self.endpoint, self.homepath + '/testlockop', self.homepath + '/testlockop_renamed', self.userid, ('test app', 'testlock')) self.storage.removefile(self.endpoint, self.homepath + '/testlockop_renamed', self.userid) From 3d83107e2a2dc83a3444a00046d15be8c9169636 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 24 Nov 2022 14:59:09 +0100 Subject: [PATCH 117/325] Use COPY instead of ADD, following Codacy advice --- wopiserver-xrootd.Dockerfile | 16 ++++++++-------- wopiserver.Dockerfile | 12 ++++++------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/wopiserver-xrootd.Dockerfile b/wopiserver-xrootd.Dockerfile index 5312a196..e53b48dd 100644 --- a/wopiserver-xrootd.Dockerfile +++ b/wopiserver-xrootd.Dockerfile @@ -11,7 +11,7 @@ LABEL maintainer="cernbox-admins@cern.ch" \ org.opencontainers.image.title="The CERNBox/IOP WOPI server" \ org.opencontainers.image.version="$VERSION" -ADD ./docker/etc/epel8.repo /etc/yum.repos.d/ +COPY ./docker/etc/epel8.repo /etc/yum.repos.d/ # prerequisites: until we need to support xrootd (even on C8), we have some EPEL dependencies, easier to install via yum/dnf; # the rest is actually installed via pip, including the xrootd python bindings @@ -36,18 +36,18 @@ RUN pip3 --default-timeout=900 install xrootd # install software RUN mkdir -p /app/core /app/bridge /test /etc/wopi /var/log/wopi -ADD ./src/* ./tools/* /app/ -ADD ./src/core/* /app/core/ -ADD ./src/bridge/* /app/bridge/ +COPY ./src/* ./tools/* /app/ +COPY ./src/core/* /app/core/ +COPY ./src/bridge/* /app/bridge/ RUN sed -i "s/WOPISERVERVERSION = 'git'/WOPISERVERVERSION = '$VERSION'/" /app/wopiserver.py RUN grep 'WOPISERVERVERSION =' /app/wopiserver.py -ADD wopiserver.conf /etc/wopi/wopiserver.defaults.conf -ADD test/*py test/*conf /test/ +COPY wopiserver.conf /etc/wopi/wopiserver.defaults.conf +COPY test/*py test/*conf /test/ # add basic custom configuration; need to contextualize -ADD ./docker/etc/*secret ./docker/etc/wopiserver.conf /etc/wopi/ +COPY ./docker/etc/*secret ./docker/etc/wopiserver.conf /etc/wopi/ #RUN mkdir /etc/certs -#ADD ./etc/*.pem /etc/certs/ if certificates shall be added +#COPY ./etc/*.pem /etc/certs/ if certificates shall be added CMD ["python3", "/app/wopiserver.py"] diff --git a/wopiserver.Dockerfile b/wopiserver.Dockerfile index 631541d0..f72cb164 100644 --- a/wopiserver.Dockerfile +++ b/wopiserver.Dockerfile @@ -19,15 +19,15 @@ RUN apk add g++ && \ # install software RUN mkdir -p /app/core /app/bridge /test /etc/wopi /var/log/wopi /var/wopi_local_storage -ADD ./src/* ./tools/* /app/ -ADD ./src/core/* /app/core/ -ADD ./src/bridge/* /app/bridge/ +COPY ./src/* ./tools/* /app/ +COPY ./src/core/* /app/core/ +COPY ./src/bridge/* /app/bridge/ RUN sed -i "s/WOPISERVERVERSION = 'git'/WOPISERVERVERSION = '$VERSION'/" /app/wopiserver.py && \ grep 'WOPISERVERVERSION =' /app/wopiserver.py -ADD wopiserver.conf /etc/wopi/wopiserver.defaults.conf -ADD test/*py test/*conf /test/ +COPY wopiserver.conf /etc/wopi/wopiserver.defaults.conf +COPY test/*py test/*conf /test/ # add basic custom configuration; need to contextualize -ADD ./docker/etc/*secret ./docker/etc/wopiserver.conf /etc/wopi/ +COPY ./docker/etc/*secret ./docker/etc/wopiserver.conf /etc/wopi/ ENTRYPOINT ["/app/wopiserver.py"] From 5b15c78621bfdfb73bb9420a35e51927c0e50fb9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 24 Nov 2022 15:11:23 +0100 Subject: [PATCH 118/325] Added shebang to CLI tools --- tools/wopilistconflicts.sh | 1 + tools/wopilistopenfiles.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/wopilistconflicts.sh b/tools/wopilistconflicts.sh index 5c6d2957..5f28bcee 100755 --- a/tools/wopilistconflicts.sh +++ b/tools/wopilistconflicts.sh @@ -1,2 +1,3 @@ +#!/usr/bin/bash curl --insecure --header "Authorization: Bearer "`cat /etc/wopi/iopsecret` https://`hostname`:8443/wopi/iop/conflicts echo diff --git a/tools/wopilistopenfiles.sh b/tools/wopilistopenfiles.sh index 4be39f33..6315b8ba 100755 --- a/tools/wopilistopenfiles.sh +++ b/tools/wopilistopenfiles.sh @@ -1,2 +1,3 @@ +#!/usr/bin/bash curl --insecure --header "Authorization: Bearer "`cat /etc/wopi/iopsecret` https://`hostname`:8443/wopi/iop/list echo From bfdf1c5d10d8bd9a593ac1778f2a2c5d7cfa9569 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 24 Nov 2022 15:56:32 +0100 Subject: [PATCH 119/325] Linting --- src/core/wopi.py | 4 ++-- src/core/wopiutils.py | 2 +- src/core/xrootiface.py | 9 +++++---- src/wopiserver.py | 5 ++--- test/test_storageiface.py | 3 ++- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index f826902a..2c3cebb3 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -261,8 +261,8 @@ def setLock(fileid, reqheaders, acctok): retrievedLock, lockHolder = utils.retrieveWopiLock(fileid, op, lock, acctok) # validate against either the given lock (RefreshLock case) or the given old lock (UnlockAndRelock case) if retrievedLock and not utils.compareWopiLocks(retrievedLock, (oldLock if oldLock else lock)): - # lock mismatch, the WOPI client is supposed to acknowledge the existing lock - # and deny access to the file in edit mode otherwise + # lock mismatch, the WOPI client is supposed to acknowledge the existing lock to start a collab session, + # or deny access to the file in edit mode otherwise return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, acctok['endpoint'], fn, 'The file is locked by %s' % (lockHolder if lockHolder != 'wopi' else 'another online editor')) # TODO cleanup 'wopi' case diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index cd19fc32..9b9f774f 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -143,7 +143,7 @@ def validateAndLogHeaders(op): # update bookkeeping of pending sessions if op.title() == 'Checkfileinfo' and session in srv.conflictsessions['pending'] and \ - time.mktime(time.strptime(srv.conflictsessions['pending'][session]['time'])) < time.time() - 300: + time.mktime(time.strptime(srv.conflictsessions['pending'][session]['time'])) < time.time() - 300: # a previously conflicted session is still around executing Checkfileinfo after 5 minutes, assume it got resolved _resolveSession(session, acctok['filename']) return acctok, None diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 3e790e18..3e50f822 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -200,10 +200,11 @@ def statx(endpoint, fileref, userid, versioninv=1): + '&mgm.pcmd=fileinfo&mgm.file.info.option=-m') try: # output looks like: - # keylength.file=35 file=/eos/.../filename size=2915 mtime=1599649863.0 ctime=1599649866.280468540 btime=1599649866.280468540 clock=0 mode=0644 - # uid=4179 gid=2763 fxid=19ab8b68 fid=430672744 ino=115607834422411264 pid=1713958 pxid=001a2726 xstype=adler xs=a2dfcdf9 - # etag="115607834422411264:a2dfcdf9" detached=0 layout=replica nstripes=2 lid=00100112 nrep=2 xattrn=sys.eos.btime xattrv=1599649866.280468540 - # uid:xxxx[username] gid:xxxx[group] tident:xxx name:username dn: prot:https host:xxxx.cern.ch domain:cern.ch geo: sudo:0 fsid=305 fsid=486 + # keylength.file=35 file=/eos/.../filename size=2915 mtime=1599649863.0 ctime=1599649866.280468540 + # btime=1599649866.280468540 clock=0 mode=0644 uid=xxxx gid=xxxx fxid=19ab8b68 fid=430672744 ino=115607834422411264 + # pid=1713958 pxid=001a2726 xstype=adler xs=a2dfcdf9 etag="115607834422411264:a2dfcdf9" detached=0 layout=replica + # nstripes=2 lid=00100112 nrep=2 xattrn=sys.eos.btime xattrv=1599649866.280468540 uid:xxxx[username] gid:xxxx[group] + # tident:xxx name:username dn: prot:https host:xxxx.cern.ch domain:cern.ch geo: sudo:0 fsid=305 fsid=486 # cf. https://gitlab.cern.ch/dss/eos/-/blob/master/archive/eosarch/utils.py kvlist = [kv.split('=') for kv in statInfo.split()] statxdata = {k: v.strip('"') for k, v in [kv for kv in kvlist if len(kv) == 2]} diff --git a/src/wopiserver.py b/src/wopiserver.py index 6367025e..6a32844e 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -77,7 +77,6 @@ class Wopi: # sets of sessions for which a lock conflict is outstanding or resolved conflictsessions = {'pending': {}, 'resolved': {}} - @classmethod def init(cls): '''Initialises the application, bails out in case of failures. Note this is not a __init__ method''' @@ -344,8 +343,8 @@ def iopOpenInApp(): if bridge.issupported(appname): try: res['app-url'], res['form-parameters'] = bridge.appopen(utils.generateWopiSrc(inode), acctok, - (appname, appurl, url_unquote_plus(req.args.get('appinturl', appurl)), req.headers.get('ApiKey')), - viewmode, usertoken) + (appname, appurl, url_unquote_plus(req.args.get('appinturl', appurl)), req.headers.get('ApiKey')), # noqa: E128 + viewmode, usertoken) except bridge.FailedOpen as foe: return foe.msg, foe.statuscode else: diff --git a/test/test_storageiface.py b/test/test_storageiface.py index 76037912..add018f0 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -350,7 +350,8 @@ def test_xattr(self): self.storage.writefile(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, databuf, None) self.storage.setxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 123, None) self.storage.setlock(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'test app', 'xattrlock') - self.storage.setxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 123, ('test app', 'xattrlock')) + self.storage.setxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', 123, + ('test app', 'xattrlock')) v = self.storage.getxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey') self.assertEqual(v, '123') self.storage.rmxattr(self.endpoint, self.homepath + '/test&xattr.txt', self.userid, 'testkey', ('test app', 'xattrlock')) From 7c2923faae76d2ced99b56f123aad29c8bea8852 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 24 Nov 2022 16:07:59 +0100 Subject: [PATCH 120/325] CI: use python 3.11 --- .github/workflows/python-app.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 63688404..a30e7374 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -16,10 +16,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip From f88184e5eaa1c39d4f1768fef4a22bc5e3a279a1 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 24 Nov 2022 16:35:29 +0100 Subject: [PATCH 121/325] Finalizing release --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afd176c1..a33c44e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ ## Changelog for the WOPI server -### ... Nov .. 2022 - v9.3.0 +### Thu Nov 24 2022 - v9.3.0 - Introduced heuristic to log which sessions are allowed - from opening a collaborative session and which ones are + to open a collaborative session and which ones are prevented by the application - Introduced support for app-aware locks in EOS (#94) - Disabled SaveAs action when user is not owner @@ -10,6 +10,7 @@ in bridged apps and in PutFile operations - Moved from LGTM to CodeQL workflow on GitHub (#100) - Introduced support for PutUserInfo +- Added support for the Microsoft "business" flow (#105) ### Mon Oct 17 2022 - v9.2.0 - Added option to use file or stream handler for logging (#91) From 8788fa6ca2e537214046db75902b21c28022411c Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 25 Nov 2022 14:38:11 +0100 Subject: [PATCH 122/325] Improved handling of folderurl and removed legacy code, cf. also cs3org/reva#3494 --- src/core/wopi.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 2c3cebb3..6cfb54ef 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -51,22 +51,18 @@ def checkFileInfo(fileid, acctok): if fsurl: fmd['FileSharingUrl'] = utils.generateUrlFromTemplate(fsurl, acctok) furl = acctok['folderurl'] - fmd['BreadcrumbFolderUrl'] = furl if furl != '/' else srv.wopiurl # the WOPI URL is a placeholder + if furl != '/': + fmd['BreadcrumbFolderUrl'] = furl if acctok['username'] == '': fmd['IsAnonymousUser'] = True fmd['UserFriendlyName'] = 'Guest ' + utils.randomString(3) - if '?path' in furl and furl[-1] != '/' and furl[-1] != '=': - # this is a subfolder of a public share, show it - fmd['BreadcrumbFolderName'] = furl[furl.find('?path'):].split('/')[-1] - else: - # this is the top level public share, which is anonymous + if furl != '/': fmd['BreadcrumbFolderName'] = 'Public share' else: fmd['IsAnonymousUser'] = False fmd['UserFriendlyName'] = acctok['username'] - fmd['BreadcrumbFolderName'] = 'Back to ' + os.path.dirname(acctok['filename']) - if furl == '/': # if no target folder URL was given, override the above and completely hide it - fmd['BreadcrumbFolderName'] = '' + if furl != '/': + fmd['BreadcrumbFolderName'] = 'Parent folder' if acctok['viewmode'] in (utils.ViewMode.READ_ONLY, utils.ViewMode.READ_WRITE) \ and srv.config.get('general', 'downloadurl', fallback=None): fmd['DownloadUrl'] = fmd['FileUrl'] = '%s?access_token=%s' % \ @@ -111,11 +107,10 @@ def checkFileInfo(fileid, acctok): # fmd['LastModifiedTime'] = datetime.fromtimestamp(int(statInfo['mtime'])).isoformat() # this currently breaks res = flask.Response(json.dumps(fmd), mimetype='application/json') - # amend sensitive metadata for the logs + # redact sensitive metadata for the logs fmd['HostViewUrl'] = fmd['HostEditUrl'] = fmd['DownloadUrl'] = fmd['FileUrl'] = \ fmd['BreadcrumbBrandUrl'] = fmd['FileSharingUrl'] = '_redacted_' - log.info('msg="File metadata response" token="%s" metadata="%s"' % - (flask.request.args['access_token'][-20:], fmd)) + log.info('msg="File metadata response" token="%s" metadata="%s"' % (flask.request.args['access_token'][-20:], fmd)) return res except IOError as e: log.info('msg="Requested file not found" filename="%s" token="%s" details="%s"' % From e4b8834a122cfb113360018841bd74f8acff1328 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 25 Nov 2022 16:22:20 +0100 Subject: [PATCH 123/325] Bridged apps: use the folderurl parameter --- src/bridge/__init__.py | 18 +++++++++++++++++- src/bridge/codimd.py | 1 + 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index e33a3df3..003c3732 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -216,7 +216,23 @@ def appopen(wopisrc, acctok, appmd, viewmode, revatok=None): # user has no write privileges, just fetch the document and push it to the app on a random docid wopilock = app.loadfromstorage(filemd, wopisrc, acctok, None) - redirurl = app.getredirecturl(viewmode, wopisrc, acctok, wopilock['doc'][1:], filemd['BaseFileName'], + # extract the path from the given folder URL: TODO this works with Reva master, not with Reva edge! + try: + filepath = urlparse.urlparse(filemd['BreadcrumbFolderUrl']).path + if filepath.find('/s/') == 0: + filepath = filepath[3:] + '/' # top of public link, no leading / + elif filepath.find('/files/public/show/') == 0: + filepath = filepath[19:] + '/' # subfolder of public link, no leading / + elif filepath.find('/files/spaces/') == 0: + filepath = filepath[13:] + '/' # direct path to resource with leading / + else: + # other folderurl strctures are not supported for the time being + filepath = "" + except (ValueError, IndexError) as e: + WB.log.warning('msg="Failed to parse folderUrl" url="%s" error="%s" token="%s"' % + (filemd['BreadcrumbFolderUrl'], e, acctok[-20:])) + filepath = "" + redirurl = app.getredirecturl(viewmode, wopisrc, acctok, wopilock['doc'][1:], filepath + filemd['BaseFileName'], filemd['UserFriendlyName'], revatok) except app.AppFailure as e: # this can be raised by loadfromstorage or getredirecturl diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index dca7aec5..23d5cd61 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -62,6 +62,7 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname, reva 'accessToken': acctok, 'disableEmbedding': ('%s' % (os.path.splitext(filename)[1] != '.zmd')).lower(), 'displayName': displayname, + 'path': os.path.dirname(filename), } if revatok: params['revaToken'] = revatok From ac8a0143057e94c35ca9393f2fee88e1d2fb3357 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 30 Nov 2022 16:10:14 +0100 Subject: [PATCH 124/325] Enable some useful postMessage properties --- src/core/wopi.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/wopi.py b/src/core/wopi.py index 6cfb54ef..416e9a01 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -13,6 +13,7 @@ import http.client from datetime import datetime from urllib.parse import unquote_plus as url_unquote +from urllib.parse import urlparse from more_itertools import peekable import flask import core.wopiutils as utils @@ -40,6 +41,9 @@ def checkFileInfo(fileid, acctok): hosteurl = srv.config.get('general', 'hostediturl', fallback=None) if hosteurl: fmd['HostEditUrl'] = utils.generateUrlFromTemplate(hosteurl, acctok) + host = urlparse(fmd['HostEditUrl']) + fmd['PostMessageOrigin'] = host.scheme + '://' + host.netloc + fmd['EditModePostMessage'] = fmd['EditNotificationPostMessage'] = True else: fmd['HostEditUrl'] = '%s%s%s' % (acctok['appediturl'], '&' if '?' in acctok['appediturl'] else '?', wopiSrc) hostvurl = srv.config.get('general', 'hostviewurl', fallback=None) @@ -50,6 +54,7 @@ def checkFileInfo(fileid, acctok): fsurl = srv.config.get('general', 'filesharingurl', fallback=None) if fsurl: fmd['FileSharingUrl'] = utils.generateUrlFromTemplate(fsurl, acctok) + fmd['FileSharingPostMessage'] = True furl = acctok['folderurl'] if furl != '/': fmd['BreadcrumbFolderUrl'] = furl From 2384b10b45ec80a459ea160c2fdd469b7f245b46 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 30 Nov 2022 17:57:43 +0100 Subject: [PATCH 125/325] xroot: ensure unlock always attempts to remove the eos-specific lock --- src/core/xrootiface.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 3e50f822..70c0f0a6 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -371,8 +371,11 @@ def unlock(endpoint, filepath, userid, appname, value): if not getlock(endpoint, filepath, userid): raise IOError(common.EXCL_ERROR) log.debug('msg="Invoked unlock" filepath="%s" value="%s"' % (filepath, value)) - rmxattr(endpoint, filepath, userid, common.LOCKKEY, (appname, None)) - rmxattr(endpoint, filepath, userid, EOSLOCKKEY, (appname, None)) + try: + rmxattr(endpoint, filepath, userid, common.LOCKKEY, (appname, None)) + finally: + # make sure this is attempted regardless the result of the previous operation + rmxattr(endpoint, filepath, userid, EOSLOCKKEY, (appname, None)) def readfile(endpoint, filepath, userid, _lockid): From 419789548dd75b25436c8517fed92f903ec1ebf5 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 30 Nov 2022 18:53:47 +0100 Subject: [PATCH 126/325] Improved logging --- src/core/wopi.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 416e9a01..8a40b288 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -580,14 +580,17 @@ def putFile(fileid, acctok): return utils.storeAfterConflict(acctok, 'External', lock, 'The file being edited got moved or overwritten') -def putUserInfo(_fileid, reqbody, acctok): +def putUserInfo(fileid, reqbody, acctok): '''Implements the PutUserInfo WOPI call''' try: statInfo = st.statx(acctok['endpoint'], acctok['filename'], acctok['userid']) lockmd = st.getlock(acctok['endpoint'], acctok['filename'], acctok['userid']) lockmd = (acctok['appname'], utils.encodeLock(lockmd)) if lockmd else None st.setxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], utils.USERINFOKEY, reqbody.decode(), lockmd) + log.info('msg="PutUserInfo" user="%s" filename="%s" fileid="%s" token="%s"' % + (acctok['userid'][-20:], acctok['filename'], fileid, flask.request.args['access_token'][-20:])) return 'OK', http.client.OK except IOError as e: - log.error('msg="PutUserInfo" token="%s" error="%s"' % (flask.request.args['access_token'][-20:], e)) + log.error('msg="PutUserInfo failed" filename="%s" error="%s" token="%s"' % + (acctok['filename'], e, flask.request.args['access_token'][-20:])) return IO_ERROR, http.client.INTERNAL_SERVER_ERROR From 361a55b74e0bed97f72a2ff85b7f379eb8234b65 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 2 Dec 2022 11:19:27 +0100 Subject: [PATCH 127/325] Bridged apps: protect from missing BreadcrumbFolderUrl --- src/bridge/__init__.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 003c3732..83101a20 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -217,21 +217,22 @@ def appopen(wopisrc, acctok, appmd, viewmode, revatok=None): wopilock = app.loadfromstorage(filemd, wopisrc, acctok, None) # extract the path from the given folder URL: TODO this works with Reva master, not with Reva edge! - try: - filepath = urlparse.urlparse(filemd['BreadcrumbFolderUrl']).path - if filepath.find('/s/') == 0: - filepath = filepath[3:] + '/' # top of public link, no leading / - elif filepath.find('/files/public/show/') == 0: - filepath = filepath[19:] + '/' # subfolder of public link, no leading / - elif filepath.find('/files/spaces/') == 0: - filepath = filepath[13:] + '/' # direct path to resource with leading / - else: - # other folderurl strctures are not supported for the time being - filepath = "" - except (ValueError, IndexError) as e: - WB.log.warning('msg="Failed to parse folderUrl" url="%s" error="%s" token="%s"' % - (filemd['BreadcrumbFolderUrl'], e, acctok[-20:])) - filepath = "" + filepath = "" + if 'BreadcrumbFolderUrl' in filemd: + try: + filepath = urlparse.urlparse(filemd['BreadcrumbFolderUrl']).path + if filepath.find('/s/') == 0: + filepath = filepath[3:] + '/' # top of public link, no leading / + elif filepath.find('/files/public/show/') == 0: + filepath = filepath[19:] + '/' # subfolder of public link, no leading / + elif filepath.find('/files/spaces/') == 0: + filepath = filepath[13:] + '/' # direct path to resource with leading / + else: + # other folderurl strctures are not supported for the time being + filepath = "" + except (ValueError, IndexError) as e: + WB.log.warning('msg="Failed to parse folderUrl, ignoring" url="%s" error="%s" token="%s"' % + (filemd['BreadcrumbFolderUrl'], e, acctok[-20:])) redirurl = app.getredirecturl(viewmode, wopisrc, acctok, wopilock['doc'][1:], filepath + filemd['BaseFileName'], filemd['UserFriendlyName'], revatok) except app.AppFailure as e: From a22cf7977056c1e570414be958c46f63041dda45 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sun, 27 Nov 2022 18:47:46 +0100 Subject: [PATCH 128/325] Implemented forced eviction of locks to compensate Microsoft Office mishandling of collaborative sessions --- src/core/wopi.py | 60 +++++++++++++++++++------------- src/core/wopiutils.py | 79 +++++++++++++++++++++++++++++++------------ src/wopiserver.py | 6 ++-- wopiserver.conf | 7 ++++ 4 files changed, 105 insertions(+), 47 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 8a40b288..b10963c8 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -166,6 +166,7 @@ def setLock(fileid, reqheaders, acctok): validateTarget = reqheaders.get('X-WOPI-Validate-Target') retrievedLock, lockHolder = utils.retrieveWopiLock(fileid, op, lock, acctok) fn = acctok['filename'] + savetime = None try: # validate that the underlying file is still there (it might have been moved/deleted) @@ -177,20 +178,21 @@ def setLock(fileid, reqheaders, acctok): return 'File not found', http.client.NOT_FOUND return IO_ERROR, http.client.INTERNAL_SERVER_ERROR - # perform the required checks for the validity of the new lock - if op == 'REFRESH_LOCK' and not retrievedLock: - if validateTarget: - # this is an extension of the API: a REFRESH_LOCK without previous lock but with a Validate-Target header - # is allowed provided that the target file was last saved by WOPI and not overwritten by external actions - # (cf. PutFile logic) - savetime = st.getxattr(acctok['endpoint'], fn, acctok['userid'], utils.LASTSAVETIMEKEY) - if savetime and (not savetime.isdigit() or int(savetime) < int(statInfo['mtime'])): - savetime = None - else: + if retrievedLock or op == 'REFRESH_LOCK': + # useful for later checks + savetime = st.getxattr(acctok['endpoint'], fn, acctok['userid'], utils.LASTSAVETIMEKEY) + if savetime and (not savetime.isdigit() or int(savetime) < int(statInfo['mtime'])): + # we had stale information, discard savetime = None - if not savetime: - return utils.makeConflictResponse(op, acctok['userid'], None, lock, oldLock, acctok['endpoint'], fn, - 'The file was not locked' + ' and got modified' if validateTarget else '') + + # perform the required checks for the validity of the new lock + if op == 'REFRESH_LOCK' and not retrievedLock and (not validateTarget or savetime): + # validateTarget is an extension of the API: a REFRESH_LOCK without previous lock but with a Validate-Target header + # is allowed, provided that the target file was last saved by WOPI (i.e. savetime is valid) and not overwritten + # by other external actions (cf. PutFile logic) + return utils.makeConflictResponse(op, acctok['userid'], None, lock, oldLock, fn, + 'The file was not locked' + (' and got modified' if validateTarget else ''), + savetime=savetime) # now check for and create an "external" lock if required if srv.config.get('general', 'detectexternallocks', fallback='True').upper() == 'TRUE' and \ @@ -198,7 +200,7 @@ def setLock(fileid, reqheaders, acctok): try: if retrievedLock == 'External': return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, - acctok['endpoint'], fn, 'The file is locked by ' + lockHolder) + fn, 'The file is locked by ' + lockHolder, savetime=savetime) # create a LibreOffice-compatible lock file for interoperability purposes, making sure to # not overwrite any existing or being created lock @@ -228,7 +230,7 @@ def setLock(fileid, reqheaders, acctok): (op.title(), fn, lockholder if lockholder else retrievedlolock)) reason = 'File locked by ' + ((lockholder + ' via LibreOffice') if lockholder else 'a LibreOffice user') return utils.makeConflictResponse(op, acctok['userid'], 'External App', lock, oldLock, - acctok['endpoint'], fn, reason) + fn, reason, savetime=savetime) # else it's our previous lock or it had expired: all right, move on else: # any other error is logged but not raised as this is optimistically not blocking WOPI operations @@ -242,7 +244,7 @@ def setLock(fileid, reqheaders, acctok): # and return conflict response if the file was already locked st.setlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(lock)) - # on first lock, set an xattr with the current time for later conflicts checking + # on first lock, set in addition an xattr with the current time for later conflicts checking try: st.setxattr(acctok['endpoint'], fn, acctok['userid'], utils.LASTSAVETIMEKEY, int(time.time()), (acctok['appname'], utils.encodeLock(lock))) @@ -263,9 +265,19 @@ def setLock(fileid, reqheaders, acctok): if retrievedLock and not utils.compareWopiLocks(retrievedLock, (oldLock if oldLock else lock)): # lock mismatch, the WOPI client is supposed to acknowledge the existing lock to start a collab session, # or deny access to the file in edit mode otherwise - return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, acctok['endpoint'], fn, - 'The file is locked by %s' % - (lockHolder if lockHolder != 'wopi' else 'another online editor')) # TODO cleanup 'wopi' case + evicted = False + if 'forcelock' in acctok and retrievedLock != 'External': + # here we try to evict the existing lock, and if possible we let the user go: + # this is to work around an issue with the Microsoft cloud! + evicted = utils.checkAndEvictLock(acctok['userid'], acctok['appname'], retrievedLock, lock, + acctok['endpoint'], fn, int(statInfo['mtime'])) + if evicted: + return utils.makeLockSuccessResponse(op, acctok, lock, 'v%s' % statInfo['etag']) + else: + return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, fn, + 'The file is locked by %s' % + (lockHolder if lockHolder != 'wopi' else 'another online editor'), # TODO cleanup 'wopi' case + savetime=savetime) # else it's our own lock, refresh it (rechecking the oldLock if necessary, for atomicity) and return try: @@ -298,7 +310,7 @@ def unlock(fileid, reqheaders, acctok): retrievedLock, _ = utils.retrieveWopiLock(fileid, 'UNLOCK', lock, acctok) if not utils.compareWopiLocks(retrievedLock, lock): return utils.makeConflictResponse('UNLOCK', acctok['userid'], retrievedLock, lock, 'NA', - acctok['endpoint'], acctok['filename'], 'Lock mismatch unlocking file') + acctok['filename'], 'Lock mismatch unlocking file') # OK, the lock matches, remove it try: # validate that the underlying file is still there @@ -387,7 +399,7 @@ def putRelative(fileid, reqheaders, acctok): 'Url': utils.generateWopiSrc(statInfo['inode'], acctok['appname'] == srv.proxiedappname), } return utils.makeConflictResponse('PUT_RELATIVE', acctok['userid'], retrievedTargetLock, 'NA', 'NA', - acctok['endpoint'], relTarget, respmd) + relTarget, respmd) except IOError: # optimistically assume we're clear pass @@ -446,7 +458,7 @@ def deleteFile(fileid, _reqheaders_unused, acctok): if retrievedLock is not None: # file is locked and cannot be deleted return utils.makeConflictResponse('DELETE', acctok['userid'], retrievedLock, 'NA', 'NA', - acctok['endpoint'], acctok['filename'], 'Cannot delete a locked file') + acctok['filename'], 'Cannot delete a locked file') try: st.removefile(acctok['endpoint'], acctok['filename'], acctok['userid']) return 'OK', http.client.OK @@ -471,7 +483,7 @@ def renameFile(fileid, reqheaders, acctok): retrievedLock, _ = utils.retrieveWopiLock(fileid, 'RENAMEFILE', lock, acctok) if retrievedLock is not None and not utils.compareWopiLocks(retrievedLock, lock): return utils.makeConflictResponse('RENAMEFILE', acctok['userid'], retrievedLock, lock, 'NA', - acctok['endpoint'], acctok['filename'], 'Lock mismatch renaming file') + acctok['filename'], 'Lock mismatch renaming file') try: # the destination name comes without base path and typically without extension targetName = os.path.dirname(acctok['filename']) + os.path.sep + targetName \ @@ -540,7 +552,7 @@ def putFile(fileid, acctok): retrievedLock, lockHolder = utils.retrieveWopiLock(fileid, 'PUTFILE', lock, acctok) if retrievedLock is None: return utils.makeConflictResponse('PUTFILE', acctok['userid'], retrievedLock, lock, 'NA', - acctok['endpoint'], acctok['filename'], 'Cannot overwrite unlocked file') + acctok['filename'], 'Cannot overwrite unlocked file') if not utils.compareWopiLocks(retrievedLock, lock): log.warning('msg="Forcing conflict based on external lock" user="%s" filename="%s" token="%s"' % (acctok['userid'][-20:], acctok['filename'], flask.request.args['access_token'][-20:])) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 9b9f774f..1ea302c2 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -192,7 +192,7 @@ def randomString(size): return ''.join([choice(ascii_lowercase) for _ in range(size)]) -def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app): +def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app, forcelock=False): '''Generates an access token for a given file and a given user, and returns a tuple with the file's inode and the URL-encoded access token.''' appname, appediturl, appviewurl = app @@ -226,16 +226,20 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app # endpoint does not set appname when the app is not proxied, so we optimistically assume it's Collabora and let it go) log.info('msg="Forcing read-only access to ODF file" filename="%s"' % statinfo['filepath']) viewmode = ViewMode.READ_ONLY - acctok = jwt.encode({'userid': userid, 'wopiuser': wopiuser, 'filename': statinfo['filepath'], 'fileid': fileid, - 'username': username, 'viewmode': viewmode.value, 'folderurl': folderurl, 'endpoint': endpoint, - 'appname': appname, 'appediturl': appediturl, 'appviewurl': appviewurl, - 'exp': exptime, 'iss': 'cs3org:wopiserver:%s' % WOPIVER}, # standard claims - srv.wopisecret, algorithm='HS256') + tokmd = { + 'userid': userid, 'wopiuser': wopiuser, 'filename': statinfo['filepath'], 'fileid': fileid, + 'username': username, 'viewmode': viewmode.value, 'folderurl': folderurl, 'endpoint': endpoint, + 'appname': appname, 'appediturl': appediturl, 'appviewurl': appviewurl, + 'exp': exptime, 'iss': 'cs3org:wopiserver:%s' % WOPIVER # standard claims + } + if forcelock: + tokmd['forcelock'] = '1' + acctok = jwt.encode(tokmd, srv.wopisecret, algorithm='HS256') log.info('msg="Access token generated" userid="%s" wopiuser="%s" mode="%s" endpoint="%s" filename="%s" inode="%s" ' - 'mtime="%s" folderurl="%s" appname="%s" expiration="%d" token="%s"' % + 'mtime="%s" folderurl="%s" appname="%s" expiration="%d" forcelock="%s" token="%s"' % (userid[-20:], wopiuser if wopiuser != userid else username, viewmode, endpoint, statinfo['filepath'], statinfo['inode'], statinfo['mtime'], - folderurl, appname, exptime, acctok[-20:])) + folderurl, appname, exptime, '1' if forcelock else '0', acctok[-20:])) return statinfo['inode'], acctok, viewmode @@ -357,30 +361,63 @@ def compareWopiLocks(lock1, lock2): return False -def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, endpoint, filename, reason=None): +def checkAndEvictLock(user, appname, retrievedlock, lock, endpoint, filename, savetime): + '''Checks if the current lock can be evicted to overcome issue with Microsoft 365 cloud''' + evictlocktime = srv.config.get('general', 'evictlocktime', fallback='') + try: + evictlocktime = int(evictlocktime) + except ValueError: + return False + session = flask.request.headers.get('X-WOPI-SessionId') + if savetime > time.time() - evictlocktime: + # file is being edited, don't evict existing lock + log.warning('msg="File is actively edited, force-unlock prevented" lockop="Lock" user="%s" ' + 'filename="%s" fileage="%1.1f" token="%s" sessionId="%s"' % + (user, filename, (time.time() - int(savetime)), flask.request.args['access_token'][-20:], session)) + return False + # ok, remove current lock and set new one + try: + st.refreshlock(endpoint, filename, user, appname, encodeLock(lock), encodeLock(retrievedlock)) + except IOError as e: + log.error('msg="Failed to force a refreshlock" user="%s" filename="%s" error="%s" token="%s" sessionId="%s"' % + (user, filename, e, flask.request.args['access_token'][-20:], session)) + return False + # and note the stealer and the evicted sessions + if session not in srv.conflictsessions['tookover']: + try: + formersession = json.loads(retrievedlock)['S'] + except (TypeError, ValueError, KeyError): + formersession = retrievedlock + srv.conflictsessions['tookover'][session] = {'time': time.asctime(), 'former': formersession} + log.warning('msg="Former session was evicted" lockop="Lock" user="%s" filename="%s" fileage="%1.1f" ' + 'formerlock="%s" token="%s" newsession="%s"' % + (user, filename, (time.time() - int(savetime)), retrievedlock, + flask.request.args['access_token'][-20:], session)) + return True + + +def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, filename, reason, savetime=None): '''Generates and logs an HTTP 409 response in case of locks conflict''' resp = flask.Response(mimetype='application/json') resp.headers['X-WOPI-Lock'] = retrievedlock if retrievedlock else '' resp.status_code = http.client.CONFLICT - if reason: - # this is either a simple message or a dictionary: in all cases we want a dictionary to be JSON-ified - if isinstance(reason, str): - reason = {'message': reason} - resp.headers['X-WOPI-LockFailureReason'] = reason['message'] - resp.data = json.dumps(reason) + if isinstance(reason, str): + # transform the given message in a dict to be JSON-ified + reason = {'message': reason} + resp.headers['X-WOPI-LockFailureReason'] = reason['message'] + resp.data = json.dumps(reason) session = flask.request.headers.get('X-WOPI-SessionId') if session and retrievedlock != 'External' and session not in srv.conflictsessions['pending']: srv.conflictsessions['pending'][session] = {'time': time.asctime(), 'held': retrievedlock} - savetime = st.getxattr(endpoint, filename, user, LASTSAVETIMEKEY) if savetime: - savetime = int(savetime) + fileage = '%1.1f' % (time.time() - int(savetime)) else: - savetime = 0 + fileage = 'NA' log.warning('msg="Returning conflict" lockop="%s" user="%s" filename="%s" token="%s" sessionId="%s" lock="%s" ' - 'oldlock="%s" retrievedlock="%s" fileage="%1.1f" reason="%s"' % + 'oldlock="%s" retrievedlock="%s" fileage="%s" reason="%s"' % (operation.title(), user, filename, flask.request.args['access_token'][-20:], - session, lock, oldlock, retrievedlock, time.time() - savetime, + session, lock, oldlock, retrievedlock, fileage, (reason['message'] if reason else 'NA'))) return resp @@ -473,7 +510,7 @@ def storeAfterConflict(acctok, retrievedlock, lock, reason): # use a CONFLICT response as it is better handled by the app to signal the issue to the user return makeConflictResponse('PUTFILE', acctok['userid'], retrievedlock, lock, 'NA', - acctok['endpoint'], acctok['filename'], reason) + acctok['filename'], reason) def storeForRecovery(content, username, filename, acctokforlog, exception): diff --git a/src/wopiserver.py b/src/wopiserver.py index 6a32844e..5380ac10 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -75,7 +75,7 @@ class Wopi: log = utils.JsonLogger(app.logger) openfiles = {} # sets of sessions for which a lock conflict is outstanding or resolved - conflictsessions = {'pending': {}, 'resolved': {}} + conflictsessions = {'pending': {}, 'resolved': {}, 'tookover': {}} @classmethod def init(cls): @@ -281,6 +281,7 @@ def iopOpenInApp(): - string appurl: the URL of the end-user application - string appviewurl (optional): the URL of the end-user application in view mode when different (defaults to appurl) - string appinturl (optional): the internal URL of the end-user application (applicable with containerized deployments) + - string forcelock (optional): if present, will force the lock when possible to work around MS Office issues Returns: a JSON response as follows: { "app-url" : "", @@ -321,6 +322,7 @@ def iopOpenInApp(): appname = url_unquote_plus(req.args.get('appname', '')) appurl = url_unquote_plus(req.args.get('appurl', '')).strip('/') appviewurl = url_unquote_plus(req.args.get('appviewurl', appurl)).strip('/') + forcelock = req.args.get('forcelock', False) if not appname or not appurl: Wopi.log.warning('msg="iopOpenInApp: app-related arguments must be provided" client="%s"' % req.remote_addr) return 'Missing appname or appurl arguments', http.client.BAD_REQUEST @@ -332,7 +334,7 @@ def iopOpenInApp(): # in this case we override the wopiuser with the resolved uid:gid wopiuser = userid inode, acctok, vm = utils.generateAccessToken(userid, fileid, viewmode, (username, wopiuser), folderurl, endpoint, - (appname, appurl, appviewurl)) + (appname, appurl, appviewurl), forcelock=forcelock) except IOError as e: Wopi.log.info('msg="iopOpenInApp: remote error on generating token" client="%s" user="%s" ' 'friendlyname="%s" mode="%s" endpoint="%s" reason="%s"' % diff --git a/wopiserver.conf b/wopiserver.conf index e04f2287..6582560e 100644 --- a/wopiserver.conf +++ b/wopiserver.conf @@ -90,6 +90,13 @@ wopilockexpiration = 1800 # on-premise setups. #wopilockstrictcheck = False +# Enable the ability to force-unlock sessions to overcome issues with Microsoft Office: +# if a session (access token) includes the option to override a previous lock, and the +# corresponding file was saved more than the given evictlocktime seconds before the request +# for lock, the previous lock is force-unlocked in order to grant a lock to the new +# session. By default this behavior is disabled. +#evictlocktime = + # Enable support of rename operations from WOPI apps. This is currently # disabled by default because the implementation is not complete, # and it is to be enabled for testing purposes only for the time being. From 8418d644380c3ee5ba964264141880513c752c1d Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 30 Nov 2022 17:44:47 +0100 Subject: [PATCH 129/325] Added additional conflict case to support storage-created locks --- src/core/wopi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index b10963c8..f6779dff 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -262,7 +262,8 @@ def setLock(fileid, reqheaders, acctok): if not retrievedLock: retrievedLock, lockHolder = utils.retrieveWopiLock(fileid, op, lock, acctok) # validate against either the given lock (RefreshLock case) or the given old lock (UnlockAndRelock case) - if retrievedLock and not utils.compareWopiLocks(retrievedLock, (oldLock if oldLock else lock)): + if not retrievedLock or not utils.compareWopiLocks(retrievedLock, (oldLock if oldLock else lock)): + # in the context of the EXCL_ERROR case, retrievedLock may be None only if the storage is holding a user lock # lock mismatch, the WOPI client is supposed to acknowledge the existing lock to start a collab session, # or deny access to the file in edit mode otherwise evicted = False From 73fa7a5c9edf2ee82adf44ccf4381ad747fcbdcb Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 2 Dec 2022 18:21:27 +0100 Subject: [PATCH 130/325] Minor fixes and improvements --- src/core/commoniface.py | 2 +- src/core/wopi.py | 4 ++-- src/core/wopiutils.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/commoniface.py b/src/core/commoniface.py index f67c6168..304c1839 100644 --- a/src/core/commoniface.py +++ b/src/core/commoniface.py @@ -73,7 +73,7 @@ def retrieverevalock(rawlock): '''Restores the JSON payload from a base64-encoded Reva lock''' try: return json.loads(urlsafe_b64decode(rawlock + '==').decode()) - except (B64Error, json.JSONDecodeError) as e: + except (B64Error, json.JSONDecodeError, UnicodeDecodeError) as e: raise IOError("Unable to parse existing lock: " + str(e)) diff --git a/src/core/wopi.py b/src/core/wopi.py index f6779dff..ead2a2ce 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -277,13 +277,13 @@ def setLock(fileid, reqheaders, acctok): else: return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, fn, 'The file is locked by %s' % - (lockHolder if lockHolder != 'wopi' else 'another online editor'), # TODO cleanup 'wopi' case + (lockHolder if lockHolder else 'another editor'), savetime=savetime) # else it's our own lock, refresh it (rechecking the oldLock if necessary, for atomicity) and return try: st.refreshlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], - utils.encodeLock(lock), utils.encodeLock(oldLock) if oldLock else None) + utils.encodeLock(lock), utils.encodeLock(oldLock)) return utils.makeLockSuccessResponse(op, acctok, lock, 'v%s' % statInfo['etag']) except IOError as rle: # this is unexpected now diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 1ea302c2..168c7ae1 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -236,10 +236,10 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app tokmd['forcelock'] = '1' acctok = jwt.encode(tokmd, srv.wopisecret, algorithm='HS256') log.info('msg="Access token generated" userid="%s" wopiuser="%s" mode="%s" endpoint="%s" filename="%s" inode="%s" ' - 'mtime="%s" folderurl="%s" appname="%s" expiration="%d" forcelock="%s" token="%s"' % + 'mtime="%s" folderurl="%s" appname="%s" %s expiration="%d" token="%s"' % (userid[-20:], wopiuser if wopiuser != userid else username, viewmode, endpoint, statinfo['filepath'], statinfo['inode'], statinfo['mtime'], - folderurl, appname, exptime, '1' if forcelock else '0', acctok[-20:])) + folderurl, appname, 'forcelock="True"' if forcelock else '', exptime, acctok[-20:])) return statinfo['inode'], acctok, viewmode @@ -373,7 +373,7 @@ def checkAndEvictLock(user, appname, retrievedlock, lock, endpoint, filename, sa # file is being edited, don't evict existing lock log.warning('msg="File is actively edited, force-unlock prevented" lockop="Lock" user="%s" ' 'filename="%s" fileage="%1.1f" token="%s" sessionId="%s"' % - (user, filename, (time.time() - int(savetime)), flask.request.args['access_token'][-20:], session)) + (user, filename, (time.time() - savetime), flask.request.args['access_token'][-20:], session)) return False # ok, remove current lock and set new one try: From f645d2bfdc6ffd12ff4597627191e6c66861dcc3 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 2 Dec 2022 18:51:03 +0100 Subject: [PATCH 131/325] xroot: handle malformed locks in refreshlock In particular let refreshlock go through, thus acting as UnlockAndRelock --- src/core/xrootiface.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 70c0f0a6..79478bcf 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -328,15 +328,13 @@ def setlock(endpoint, filepath, userid, appname, value, recurse=False): setxattr(endpoint, filepath, userid, EOSLOCKKEY, _geneoslock(appname) + '&mgm.option=c', None) setxattr(endpoint, filepath, userid, common.LOCKKEY, common.genrevalock(appname, value), (appname, None)) except IOError as e: - if common.EXCL_ERROR in str(e): - # check for pre-existing stale locks (this is now not atomic) - if not getlock(endpoint, filepath, userid) and not recurse: - setlock(endpoint, filepath, userid, appname, value, recurse=True) - else: - # the lock is valid - raise + if common.EXCL_ERROR not in str(e): + raise + # check for pre-existing stale locks (this is now not atomic) + if not getlock(endpoint, filepath, userid) and not recurse: + setlock(endpoint, filepath, userid, appname, value, recurse=True) else: - # we got a different remote error, raise it + # the lock is valid raise @@ -357,7 +355,14 @@ def getlock(endpoint, filepath, userid): def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): '''Refresh the lock value as an xattr''' - currlock = getlock(endpoint, filepath, userid) + try: + currlock = getlock(endpoint, filepath, userid) + except IOError as e: + if 'Unable to parse' in e: + # ensure we can set the new lock + currlock = {'lock_id': oldvalue} + else: + raise if not currlock or (oldvalue and currlock['lock_id'] != oldvalue): raise IOError(common.EXCL_ERROR) log.debug('msg="Invoked refreshlock" filepath="%s" value="%s"' % (filepath, value)) From 509ae52ff90f2a8c99c585a26cfe918378a90955 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 6 Dec 2022 16:29:46 +0100 Subject: [PATCH 132/325] Fixed logging --- src/bridge/codimd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 23d5cd61..84631d44 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -180,7 +180,7 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): verify=sslverify) if res.status_code == http.client.FORBIDDEN: # the file got unlocked because of no activity, yet some user is there: let it go - log.warning('msg="Document was being edited in CodiMD, redirecting user" token"%s"' % acctok[-20:]) + log.warning('msg="Document was being edited in CodiMD, redirecting user" token="%s"' % acctok[-20:]) elif res.status_code == http.client.REQUEST_ENTITY_TOO_LARGE: log.error('msg="File is too large to be edited in CodiMD" docid="%s" token="%s"' % (docid, acctok[-20:])) raise AppFailure(TOOLARGE) From 4d8d4f8db1facddc07f129cb53e6a78ffa71434e Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 8 Dec 2022 11:33:48 +0100 Subject: [PATCH 133/325] Improved logging --- src/bridge/wopiclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bridge/wopiclient.py b/src/bridge/wopiclient.py index 2473834b..7f5962b7 100644 --- a/src/bridge/wopiclient.py +++ b/src/bridge/wopiclient.py @@ -163,8 +163,8 @@ def relock(wopisrc, acctok, docid, isclose): # first get again the file metadata res = request(wopisrc, acctok, 'GET') if res.status_code != http.client.OK: - log.warning('msg="Session expired or file renamed when attempting to relock it" response="%d" token="%s"' % - (res.status_code, acctok[-20:])) + log.warning('msg="Session expired or file renamed when attempting to relock it" response="%d" docid="%s" token="%s"' % + (res.status_code, docid, acctok[-20:])) raise InvalidLock('Session expired, please refresh this page') filemd = res.json() From bc18965b9ba006eba7aea74d8efa9e8ba7af4f97 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 10 Dec 2022 15:14:57 +0100 Subject: [PATCH 134/325] Improved stats for conflicted sessions --- src/core/wopiutils.py | 23 ++++++++++++++++++----- src/wopiserver.py | 4 +++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 168c7ae1..7965bcf8 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -143,7 +143,7 @@ def validateAndLogHeaders(op): # update bookkeeping of pending sessions if op.title() == 'Checkfileinfo' and session in srv.conflictsessions['pending'] and \ - time.mktime(time.strptime(srv.conflictsessions['pending'][session]['time'])) < time.time() - 300: + int(srv.conflictsessions['pending'][session]['time']) < time.time() - 300: # a previously conflicted session is still around executing Checkfileinfo after 5 minutes, assume it got resolved _resolveSession(session, acctok['filename']) return acctok, None @@ -235,6 +235,8 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app if forcelock: tokmd['forcelock'] = '1' acctok = jwt.encode(tokmd, srv.wopisecret, algorithm='HS256') + if 'MS 365' in appname: + srv.allusers.add(userid) log.info('msg="Access token generated" userid="%s" wopiuser="%s" mode="%s" endpoint="%s" filename="%s" inode="%s" ' 'mtime="%s" folderurl="%s" appname="%s" %s expiration="%d" token="%s"' % (userid[-20:], wopiuser if wopiuser != userid else username, viewmode, endpoint, @@ -388,7 +390,11 @@ def checkAndEvictLock(user, appname, retrievedlock, lock, endpoint, filename, sa formersession = json.loads(retrievedlock)['S'] except (TypeError, ValueError, KeyError): formersession = retrievedlock - srv.conflictsessions['tookover'][session] = {'time': time.asctime(), 'former': formersession} + srv.conflictsessions['tookover'][session] = { + 'time': int(time.time()), + 'user': user, + 'former': formersession + } log.warning('msg="Former session was evicted" lockop="Lock" user="%s" filename="%s" fileage="%1.1f" ' 'formerlock="%s" token="%s" newsession="%s"' % (user, filename, (time.time() - int(savetime)), retrievedlock, @@ -409,7 +415,11 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, filename session = flask.request.headers.get('X-WOPI-SessionId') if session and retrievedlock != 'External' and session not in srv.conflictsessions['pending']: - srv.conflictsessions['pending'][session] = {'time': time.asctime(), 'held': retrievedlock} + srv.conflictsessions['pending'][session] = { + 'user': user, + 'time': int(time.time()), + 'heldby': retrievedlock + } if savetime: fileage = '%1.1f' % (time.time() - int(savetime)) else: @@ -426,8 +436,11 @@ def _resolveSession(session, filename): '''Mark a session as resolved and account the given filename in the openfiles map. This is only used for bookkeeping, no functionality is associated to those maps''' if session in srv.conflictsessions['pending']: - srv.conflictsessions['pending'].pop(session) - srv.conflictsessions['resolved'][session] = time.asctime() + s = srv.conflictsessions['pending'].pop(session) + srv.conflictsessions['resolved'][session] = { + 'user': s['user'], + 'restime': int(time.time() - int(s['time'])) + } # keep some accounting of the open files if filename not in srv.openfiles: srv.openfiles[filename] = (time.asctime(), set()) diff --git a/src/wopiserver.py b/src/wopiserver.py index 5380ac10..28cd3d1a 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -75,7 +75,8 @@ class Wopi: log = utils.JsonLogger(app.logger) openfiles = {} # sets of sessions for which a lock conflict is outstanding or resolved - conflictsessions = {'pending': {}, 'resolved': {}, 'tookover': {}} + conflictsessions = {'pending': {}, 'resolved': {}, 'tookover': {}, 'users': 0} + allusers = set() @classmethod def init(cls): @@ -408,6 +409,7 @@ def iopGetConflicts(): return UNAUTHORIZED # dump the current sets in JSON format Wopi.log.info('msg="iopGetConflicts: returning outstanding/resolved conflicted sessions" client="%s"' % req.remote_addr) + Wopi.conflictsessions['users'] = len(Wopi.allusers) return flask.Response(json.dumps(Wopi.conflictsessions), mimetype='application/json') From ea590eb6aad0c523f75d02147e692382f08df66f Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 10 Dec 2022 23:33:58 +0100 Subject: [PATCH 135/325] Added logging in case of detected external modifications --- src/core/wopi.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index ead2a2ce..edcf0a27 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -183,6 +183,8 @@ def setLock(fileid, reqheaders, acctok): savetime = st.getxattr(acctok['endpoint'], fn, acctok['userid'], utils.LASTSAVETIMEKEY) if savetime and (not savetime.isdigit() or int(savetime) < int(statInfo['mtime'])): # we had stale information, discard + log.warning('msg="Detected external modification" filename="%s" savetime="%s" mtime="%s" token="%s"' % + (fn, savetime, statInfo['mtime'], flask.request.args['access_token'][-20:])) savetime = None # perform the required checks for the validity of the new lock @@ -190,6 +192,8 @@ def setLock(fileid, reqheaders, acctok): # validateTarget is an extension of the API: a REFRESH_LOCK without previous lock but with a Validate-Target header # is allowed, provided that the target file was last saved by WOPI (i.e. savetime is valid) and not overwritten # by other external actions (cf. PutFile logic) + log.debug('msg="Debug conflict" retrievedlock="%s" validateTarget="%s" savetime="%s" mtime="%s"' % + (retrievedLock, validateTarget, savetime, statInfo['mtime'])) return utils.makeConflictResponse(op, acctok['userid'], None, lock, oldLock, fn, 'The file was not locked' + (' and got modified' if validateTarget else ''), savetime=savetime) @@ -588,7 +592,7 @@ def putFile(fileid, acctok): # no xattr was there or we got our xattr but mtime is more recent: someone may have updated the file # from a different source (e.g. FUSE or SMB mount), therefore force conflict and return failure to the application - log.warning('msg="Forcing conflict based on save time" user="%s" filename="%s" savetime="%s" lastmtime="%s" token="%s"' % + log.warning('msg="Forcing conflict based on save time" user="%s" filename="%s" savetime="%s" mtime="%s" token="%s"' % (acctok['userid'][-20:], acctok['filename'], savetime, mtime, flask.request.args['access_token'][-20:])) return utils.storeAfterConflict(acctok, 'External', lock, 'The file being edited got moved or overwritten') From 0c8d59f4f33bf36b90c53904e43e77eb604806b9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sun, 11 Dec 2022 14:17:39 +0100 Subject: [PATCH 136/325] Bridged apps: fixed RefreshLock with validateTarget --- src/core/wopi.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index edcf0a27..e9e00c5c 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -188,12 +188,10 @@ def setLock(fileid, reqheaders, acctok): savetime = None # perform the required checks for the validity of the new lock - if op == 'REFRESH_LOCK' and not retrievedLock and (not validateTarget or savetime): + if op == 'REFRESH_LOCK' and not retrievedLock and (not validateTarget or not savetime): # validateTarget is an extension of the API: a REFRESH_LOCK without previous lock but with a Validate-Target header # is allowed, provided that the target file was last saved by WOPI (i.e. savetime is valid) and not overwritten # by other external actions (cf. PutFile logic) - log.debug('msg="Debug conflict" retrievedlock="%s" validateTarget="%s" savetime="%s" mtime="%s"' % - (retrievedLock, validateTarget, savetime, statInfo['mtime'])) return utils.makeConflictResponse(op, acctok['userid'], None, lock, oldLock, fn, 'The file was not locked' + (' and got modified' if validateTarget else ''), savetime=savetime) From bfe5ddc36e7e2bba642ff80fc0af11fc59469a4c Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sun, 11 Dec 2022 14:58:05 +0100 Subject: [PATCH 137/325] CI: moved release builds from Drone to GitHub actions --- .drone.yml | 100 ------------------ .../{python-app.yml => ci-tests.yml} | 16 +-- .github/workflows/codeql.yml | 4 +- .github/workflows/release.yml | 37 +++++++ README.md | 7 +- 5 files changed, 52 insertions(+), 112 deletions(-) delete mode 100644 .drone.yml rename .github/workflows/{python-app.yml => ci-tests.yml} (87%) create mode 100644 .github/workflows/release.yml diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 3e05dc31..00000000 --- a/.drone.yml +++ /dev/null @@ -1,100 +0,0 @@ ---- -kind: pipeline -type: docker -name: release-latest - -platform: - os: linux - arch: amd64 - -trigger: - branch: - - master - event: - exclude: - - pull_request - - tag - - promote - - rollback - -steps: -- name: publish-docker-wopi-latest - pull: always - image: plugins/docker - settings: - repo: cs3org/wopiserver - tags: latest - dockerfile: wopiserver.Dockerfile - username: - from_secret: dockerhub_username - password: - from_secret: dockerhub_password - build_args: - - VERSION=${DRONE_SEMVER_SHORT}-g${DRONE_COMMIT:0:7} - custom_dns: - - 128.142.17.5 - - 128.142.16.5 - ---- -kind: pipeline -type: docker -name: release - -platform: - os: linux - arch: amd64 - -trigger: - event: - include: - - tag - -steps: -- name: publish-docker-wopi-tag - pull: always - image: plugins/docker - settings: - repo: cs3org/wopiserver - tags: ${DRONE_TAG} - dockerfile: wopiserver.Dockerfile - username: - from_secret: dockerhub_username - password: - from_secret: dockerhub_password - build_args: - - VERSION=${DRONE_TAG} - custom_dns: - - 128.142.17.5 - - 128.142.16.5 - ---- -kind: pipeline -type: docker -name: release-xrootd - -platform: - os: linux - arch: amd64 - -trigger: - event: - include: - - tag - -steps: -- name: publish-docker-wopi-tag - pull: always - image: plugins/docker - settings: - repo: cs3org/wopiserver - tags: ${DRONE_TAG}-xrootd - dockerfile: wopiserver-xrootd.Dockerfile - username: - from_secret: dockerhub_username - password: - from_secret: dockerhub_password - build_args: - - VERSION=${DRONE_TAG} - custom_dns: - - 128.142.17.5 - - 128.142.16.5 diff --git a/.github/workflows/python-app.yml b/.github/workflows/ci-tests.yml similarity index 87% rename from .github/workflows/python-app.yml rename to .github/workflows/ci-tests.yml index a30e7374..0c4fa51d 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/ci-tests.yml @@ -1,36 +1,38 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Python application - +name: Linting and unit tests on: push: - branches: [ master ] + branches: [ "master", "cernbox" ] pull_request: - branches: [ master ] + branches: [ "master", "cernbox" ] jobs: build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python 3.11 uses: actions/setup-python@v2 with: python-version: "3.11" + - name: Install dependencies run: | python -m pip install --upgrade pip pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide, we further relax this flake8 . --count --exit-zero --max-complexity=30 --max-line-length=130 --statistics + - name: Test with pytest run: | pytest diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c4bcb6e1..544d5a36 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,8 +1,8 @@ -name: "CodeQL" +name: CodeQL on: push: - branches: [ "master", "aarnet", "cernbox", "collaboratest", "eoslock", "nginx", "oolock-autoexpire", "ooracefix", "open-for-iop", "perma-inode", "python3", "snyk-fix-0d459d1e2b96d3ef1c5a96800328793e", "snyk-fix-af3411a9b6ac485b96ed2d8f434e9d97", "syslog", "tus-uploads", "wopi4", "xroot-cs9" ] + branches: [ "master", "cernbox" ] pull_request: branches: [ "master" ] schedule: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..16b0131e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,37 @@ +name: Releases +on: + push: + tags: + - "*" + +jobs: + release: + runs-on: self-hosted + strategy: + matrix: + include: + - tag: wopiserver:${{ github.ref_type }} + file: wopiserver.Dockerfile + - tag: wopiserver:${{ github.ref_type }}-xrootd + file: wopiserver-xrootd.Dockerfile + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + if: ${{ github.event_name != 'pull_request' }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push ${{ matrix.tag }} + uses: docker/build-push-action@v3 + with: + context: . + push: true + file: ${{ matrix.file }} + tags: ${{ format('{0}/{1}', secrets.DOCKERHUB_ORGANIZATION, matrix.tag) }} diff --git a/README.md b/README.md index 9f7c7e88..92409a1c 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,18 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) - [![Gitter chat](https://badges.gitter.im/cs3org/wopiserver.svg)](https://gitter.im/cs3org/wopiserver) [![Build Status](https://drone.cernbox.cern.ch/api/badges/cs3org/wopiserver/status.svg)](https://drone.cernbox.cern.ch/cs3org/wopiserver) + [![Gitter chat](https://badges.gitter.im/cs3org/wopiserver.svg)](https://gitter.im/cs3org/wopiserver) + [![Build Status](https://github.com/cs3org/wopiserver/actions/workflows/releases.yml/badge.svg)](https://github.com/cs3org/wopiserver/actions) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/e4e7c46c39b04bddbf63ade4cacdcc7d)](https://www.codacy.com/gh/cs3org/wopiserver/dashboard?utm_source=github.com&utm_medium=referral&utm_content=cs3org/wopiserver&utm_campaign=Badge_Grade) [![codecov](https://codecov.io/gh/cs3org/wopiserver/branch/master/graph/badge.svg)](https://codecov.io/gh/cs3org/wopiserver) ======== # WOPI Server -This service is part of the ScienceMesh Interoperability Platform (IOP) and implements a vendor-neutral application gateway compatible with the Web-application Open Platform Interface ([WOPI](https://docs.microsoft.com/en-us/microsoft-365/cloud-storage-partner-program/online)) specifications. +This service is part of the ScienceMesh Interoperability Platform ([IOP](https://developer.sciencemesh.io)) and implements a vendor-neutral application gateway compatible with the Web-application Open Platform Interface ([WOPI](https://docs.microsoft.com/en-us/microsoft-365/cloud-storage-partner-program/online)) specifications. It enables ScienceMesh EFSS storages to integrate Office Online platforms including Microsoft Office Online and Collabora Online. In addition it implements a [bridge](src/bridge/readme.md) module with dedicated extensions to support apps like CodiMD and Etherpad. Author: Giuseppe Lo Presti (@glpatcern)
-Contributors: +Contributors (oldest contributions first): - Michael DSilva (@madsi1m) - Lovisa Lugnegaard (@LovisaLugnegard) - Samuel Alfageme (@SamuAlfageme) From 39fe7246c13388e95a042e252b20f2152c934e72 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 12 Dec 2022 11:35:29 +0100 Subject: [PATCH 138/325] CI: fixing release step --- .github/workflows/release.yml | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 16b0131e..a1624637 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,9 +10,9 @@ jobs: strategy: matrix: include: - - tag: wopiserver:${{ github.ref_type }} + - tag: wopiserver:${{ github.ref_name }} file: wopiserver.Dockerfile - - tag: wopiserver:${{ github.ref_type }}-xrootd + - tag: wopiserver:${{ github.ref_name }}-xrootd file: wopiserver-xrootd.Dockerfile steps: diff --git a/README.md b/README.md index 92409a1c..1032608b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Gitter chat](https://badges.gitter.im/cs3org/wopiserver.svg)](https://gitter.im/cs3org/wopiserver) - [![Build Status](https://github.com/cs3org/wopiserver/actions/workflows/releases.yml/badge.svg)](https://github.com/cs3org/wopiserver/actions) + [![Build Status](https://github.com/cs3org/wopiserver/actions/workflows/release.yml/badge.svg)](https://github.com/cs3org/wopiserver/actions) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/e4e7c46c39b04bddbf63ade4cacdcc7d)](https://www.codacy.com/gh/cs3org/wopiserver/dashboard?utm_source=github.com&utm_medium=referral&utm_content=cs3org/wopiserver&utm_campaign=Badge_Grade) [![codecov](https://codecov.io/gh/cs3org/wopiserver/branch/master/graph/badge.svg)](https://codecov.io/gh/cs3org/wopiserver) ======== From 031621a006c9b58fdd81f2603cdbe257e69caff9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 12 Dec 2022 17:36:04 +0100 Subject: [PATCH 139/325] Fixed resolved sessions accounting in Unlock --- src/core/wopi.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index e9e00c5c..a8c60db8 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -339,8 +339,11 @@ def unlock(fileid, reqheaders, acctok): del srv.openfiles[acctok['filename']] session = flask.request.headers.get('X-WOPI-SessionId') if session in srv.conflictsessions['pending']: - srv.conflictsessions['pending'].pop(session) - srv.conflictsessions['resolved'][session] = time.asctime() + s = srv.conflictsessions['pending'].pop(session) + srv.conflictsessions['resolved'][session] = { + 'user': s['user'], + 'restime': int(time.time() - int(s['time'])) + } except KeyError: # already removed? pass From 2f50614850bca48d4b0511a776a39ecd77a624b6 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 12 Dec 2022 17:57:10 +0100 Subject: [PATCH 140/325] Fixed "use before assignment" bug --- src/wopiserver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/wopiserver.py b/src/wopiserver.py index 28cd3d1a..8d70a5cb 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -568,8 +568,7 @@ def cboxOpen_deprecated(): if ruid == 0 or rgid == 0: raise ValueError except ValueError: - Wopi.log.warning('msg="cboxOpen: invalid or missing user/token in request" client="%s" user="%s"' % - (req.remote_addr, userid)) + Wopi.log.warning('msg="cboxOpen: invalid or missing user/token in request" client="%s"' % req.remote_addr) return UNAUTHORIZED filename = url_unquote_plus(req.args.get('filename', '')) if filename == '': From e0f0233780d4e2490e392f7bc14a6e3b05f3213d Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 12 Dec 2022 17:57:46 +0100 Subject: [PATCH 141/325] xroot: removed unneeded global statements --- src/core/xrootiface.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 79478bcf..f0d5ca8c 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -52,8 +52,6 @@ def init(inconfig, inlog): def _getxrdfor(endpoint): '''Look up the xrootd client for the given endpoint, create it if missing. Supports "default" for the defaultstorage endpoint.''' - global xrdfs # pylint: disable=global-statement - global defaultstorage # pylint: disable=global-statement if endpointoverride: endpoint = endpointoverride if endpoint == 'default': From f09bcdd561ac2a6c284b08d4232c6beadd53025a Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 12 Dec 2022 18:00:38 +0100 Subject: [PATCH 142/325] xroot: fixed exception handling --- src/core/xrootiface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index f0d5ca8c..b4585886 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -356,7 +356,7 @@ def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): try: currlock = getlock(endpoint, filepath, userid) except IOError as e: - if 'Unable to parse' in e: + if 'Unable to parse' in str(e): # ensure we can set the new lock currlock = {'lock_id': oldvalue} else: From 5d233b8437e7cb1ff46c15396184848cc10feac8 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 19 Dec 2022 11:49:17 +0100 Subject: [PATCH 143/325] CI: make sure the VERSION env is set when building a release --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a1624637..74530b83 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,6 +30,8 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push ${{ matrix.tag }} uses: docker/build-push-action@v3 + env: + VERSION: ${{ github.ref_name }} with: context: . push: true From 329863debeea949ddc0e78278f7a6061d6457ea9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 26 Dec 2022 10:37:20 +0100 Subject: [PATCH 144/325] Etherpad: do not call setEFSSMetadata on read-only pads --- src/bridge/etherpad.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/bridge/etherpad.py b/src/bridge/etherpad.py index dc68c28d..622fd07a 100644 --- a/src/bridge/etherpad.py +++ b/src/bridge/etherpad.py @@ -67,11 +67,16 @@ def _apicall(method, params, data=None, acctok=None, raiseonnonzerocode=True): def getredirecturl(viewmode, wopisrc, acctok, docid, _filename, displayname): '''Return a valid URL to the app for the given WOPI context''' + if viewmode in (utils.ViewMode.READ_ONLY, utils.ViewMode.VIEW_ONLY): + # for read-only mode generate a read-only link + res = _apicall('getReadOnlyID', {'padID': docid}, acctok=acctok) + return appexturl + '/p/%s?userName=%s' % (res['data']['readOnlyID'], urlparse.quote_plus(displayname)) + # pass to Etherpad the required metadata for the save webhook try: res = requests.post(appurl + '/setEFSSMetadata', - params={'padID': docid, 'wopiSrc': urlparse.quote_plus(wopisrc), 'accessToken': acctok, - 'apikey': apikey}, + params={'padID': docid, 'wopiSrc': urlparse.quote_plus(wopisrc), + 'accessToken': acctok, 'apikey': apikey}, verify=sslverify) if res.status_code != http.client.OK or res.json()['code'] != 0: log.error('msg="Failed to call Etherpad" method="setEFSSMetadata" token="%s" response="%d: %s"' % @@ -82,11 +87,7 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, _filename, displayname): log.error('msg="Exception raised attempting to connect to Etherpad" method="setEFSSMetadata" exception="%s"' % e) raise AppFailure - if viewmode in (utils.ViewMode.READ_ONLY, utils.ViewMode.VIEW_ONLY): - # for read-only mode generate a read-only link - res = _apicall('getReadOnlyID', {'padID': docid}, acctok=acctok) - return appexturl + '/p/%s?userName=%s' % (res['data']['readOnlyID'], urlparse.quote_plus(displayname)) - # return the URL to the pad (TODO if viewmode is PREVIEW) + # return the URL to the pad for editing (a PREVIEW viewmode is not supported) return appexturl + '/p/%s?userName=%s' % (docid, urlparse.quote_plus(displayname)) From 2cc3abe7276acd545b7753db38645beebf7dd7db Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 3 Jan 2023 22:09:09 +0100 Subject: [PATCH 145/325] More logging and tracking of evicted locks --- src/core/wopi.py | 6 +++--- src/core/wopiutils.py | 22 ++++++++++++++-------- src/wopiserver.py | 2 +- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index a8c60db8..a6710b41 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -263,16 +263,16 @@ def setLock(fileid, reqheaders, acctok): # get the lock that was set if not retrievedLock: retrievedLock, lockHolder = utils.retrieveWopiLock(fileid, op, lock, acctok) - # validate against either the given lock (RefreshLock case) or the given old lock (UnlockAndRelock case) + # validate against either the given lock (RefreshLock case) or the given old lock (UnlockAndRelock case); + # in the context of the EXCL_ERROR case, retrievedLock may be None only if the storage is holding a user lock if not retrievedLock or not utils.compareWopiLocks(retrievedLock, (oldLock if oldLock else lock)): - # in the context of the EXCL_ERROR case, retrievedLock may be None only if the storage is holding a user lock # lock mismatch, the WOPI client is supposed to acknowledge the existing lock to start a collab session, # or deny access to the file in edit mode otherwise evicted = False if 'forcelock' in acctok and retrievedLock != 'External': # here we try to evict the existing lock, and if possible we let the user go: # this is to work around an issue with the Microsoft cloud! - evicted = utils.checkAndEvictLock(acctok['userid'], acctok['appname'], retrievedLock, lock, + evicted = utils.checkAndEvictLock(acctok['userid'], acctok['appname'], retrievedLock, oldLock, lock, acctok['endpoint'], fn, int(statInfo['mtime'])) if evicted: return utils.makeLockSuccessResponse(op, acctok, lock, 'v%s' % statInfo['etag']) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 7965bcf8..32cea911 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -363,7 +363,7 @@ def compareWopiLocks(lock1, lock2): return False -def checkAndEvictLock(user, appname, retrievedlock, lock, endpoint, filename, savetime): +def checkAndEvictLock(user, appname, retrievedlock, oldlock, lock, endpoint, filename, savetime): '''Checks if the current lock can be evicted to overcome issue with Microsoft 365 cloud''' evictlocktime = srv.config.get('general', 'evictlocktime', fallback='') try: @@ -373,9 +373,15 @@ def checkAndEvictLock(user, appname, retrievedlock, lock, endpoint, filename, sa session = flask.request.headers.get('X-WOPI-SessionId') if savetime > time.time() - evictlocktime: # file is being edited, don't evict existing lock - log.warning('msg="File is actively edited, force-unlock prevented" lockop="Lock" user="%s" ' + log.warning('msg="File is actively edited, force-unlock prevented" lockop="%s" user="%s" ' 'filename="%s" fileage="%1.1f" token="%s" sessionId="%s"' % - (user, filename, (time.time() - savetime), flask.request.args['access_token'][-20:], session)) + (('UnlockAndRelock' if oldlock else 'Lock'), user, filename, (time.time() - int(savetime)), + flask.request.args['access_token'][-20:], session)) + if session: + srv.conflictsessions['failedtotakeover'][session] = { + 'time': int(time.time()), + 'user': user + } return False # ok, remove current lock and set new one try: @@ -395,10 +401,10 @@ def checkAndEvictLock(user, appname, retrievedlock, lock, endpoint, filename, sa 'user': user, 'former': formersession } - log.warning('msg="Former session was evicted" lockop="Lock" user="%s" filename="%s" fileage="%1.1f" ' + log.warning('msg="Former session was evicted" lockop="%s" user="%s" filename="%s" fileage="%1.1f" ' 'formerlock="%s" token="%s" newsession="%s"' % - (user, filename, (time.time() - int(savetime)), retrievedlock, - flask.request.args['access_token'][-20:], session)) + (('UnlockAndRelock' if oldlock else 'Lock'), user, filename, (time.time() - int(savetime)), + retrievedlock, flask.request.args['access_token'][-20:], session)) return True @@ -426,8 +432,8 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, filename fileage = 'NA' log.warning('msg="Returning conflict" lockop="%s" user="%s" filename="%s" token="%s" sessionId="%s" lock="%s" ' 'oldlock="%s" retrievedlock="%s" fileage="%s" reason="%s"' % - (operation.title(), user, filename, flask.request.args['access_token'][-20:], - session, lock, oldlock, retrievedlock, fileage, + (('UnlockAndRelock' if oldlock else operation.title()), user, filename, + flask.request.args['access_token'][-20:], session, lock, oldlock, retrievedlock, fileage, (reason['message'] if reason else 'NA'))) return resp diff --git a/src/wopiserver.py b/src/wopiserver.py index 8d70a5cb..8b9a876b 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -75,7 +75,7 @@ class Wopi: log = utils.JsonLogger(app.logger) openfiles = {} # sets of sessions for which a lock conflict is outstanding or resolved - conflictsessions = {'pending': {}, 'resolved': {}, 'tookover': {}, 'users': 0} + conflictsessions = {'pending': {}, 'resolved': {}, 'tookover': {}, 'failedtotakeover': {}, 'users': 0} allusers = set() @classmethod From fb706993d1faf04688c2cd6cb01de3a85b75ec36 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 5 Jan 2023 14:56:34 +0100 Subject: [PATCH 146/325] Bridged apps: fixed signature of getredirecturl --- src/bridge/codimd.py | 2 +- src/bridge/etherpad.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 84631d44..ee109983 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -53,7 +53,7 @@ def init(_appurl, _appinturl, _apikey): raise AppFailure -def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname, revatok=None): +def getredirecturl(viewmode, wopisrc, acctok, docid, filename, displayname, revatok): '''Return a valid URL to the app for the given WOPI context''' if viewmode in (utils.ViewMode.READ_WRITE, utils.ViewMode.PREVIEW): mode = 'view' if viewmode == utils.ViewMode.PREVIEW else 'both' diff --git a/src/bridge/etherpad.py b/src/bridge/etherpad.py index 622fd07a..837b724f 100644 --- a/src/bridge/etherpad.py +++ b/src/bridge/etherpad.py @@ -65,7 +65,7 @@ def _apicall(method, params, data=None, acctok=None, raiseonnonzerocode=True): return res -def getredirecturl(viewmode, wopisrc, acctok, docid, _filename, displayname): +def getredirecturl(viewmode, wopisrc, acctok, docid, _filename, displayname, _revatok): '''Return a valid URL to the app for the given WOPI context''' if viewmode in (utils.ViewMode.READ_ONLY, utils.ViewMode.VIEW_ONLY): # for read-only mode generate a read-only link From 8e002adbd7636e355274237489f91a98237f872d Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 9 Jan 2023 14:04:08 +0100 Subject: [PATCH 147/325] xroot: cleaner logs to help conversion to JSON --- src/core/xrootiface.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index b4585886..bdf0f21c 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -124,15 +124,15 @@ def _xrootcmd(endpoint, cmd, subcmd, userid, args, app='wopi'): msg = res[1][res[1].find('=') + 1:].strip('\n') if common.ENOENT_MSG.lower() in msg or 'unable to get attribute' in msg or rc == '2': log.info('msg="Invoked cmd on non-existing entity" cmd="%s" subcmd="%s" args="%s" result="%s" rc="%s"' % - (cmd, subcmd, args, msg.replace('error:', ''), rc.strip('\00'))) + (cmd, subcmd, args, msg.replace('error:', '').strip(), rc.strip('\00'))) raise IOError(common.ENOENT_MSG) if EXCL_XATTR_MSG in msg: log.info('msg="Invoked setxattr on an already locked entity" args="%s" result="%s" rc="%s"' % - (args, msg.replace('error:', ''), rc.strip('\00'))) + (args, msg.replace('error:', '').strip(), rc.strip('\00'))) raise IOError(common.EXCL_ERROR) if LOCK_MISMATCH_MSG or FOREIGN_XATTR_MSG in msg: log.info('msg="Mismatched lock" cmd="%s" subcmd="%s" args="%s" app="%s" result="%s" rc="%s"' % - (cmd, subcmd, args, app, msg.replace('error:', ''), rc.strip('\00'))) + (cmd, subcmd, args, app, msg.replace('error:', '').strip(), rc.strip('\00'))) raise IOError(common.EXCL_ERROR) # anything else (including permission errors) are logged as errors log.error('msg="Error with xroot" cmd="%s" subcmd="%s" args="%s" error="%s" rc="%s"' % From 49888cbfe11222c4f529c1d36f522be01bd88afb Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 10 Jan 2023 13:36:16 +0100 Subject: [PATCH 148/325] Include username in PutUserInfo key as required by specs --- src/core/wopi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index a6710b41..34c56a33 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -94,7 +94,8 @@ def checkFileInfo(fileid, acctok): fmd['SupportsRename'] = fmd['UserCanRename'] = enablerename and (acctok['viewmode'] == utils.ViewMode.READ_WRITE) fmd['SupportsContainers'] = False # TODO this is all to be implemented fmd['SupportsUserInfo'] = True - uinfo = st.getxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], utils.USERINFOKEY) + uinfo = st.getxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], + utils.USERINFOKEY + '.' + acctok['username']) if uinfo: fmd['UserInfo'] = uinfo @@ -604,7 +605,8 @@ def putUserInfo(fileid, reqbody, acctok): statInfo = st.statx(acctok['endpoint'], acctok['filename'], acctok['userid']) lockmd = st.getlock(acctok['endpoint'], acctok['filename'], acctok['userid']) lockmd = (acctok['appname'], utils.encodeLock(lockmd)) if lockmd else None - st.setxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], utils.USERINFOKEY, reqbody.decode(), lockmd) + st.setxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], + utils.USERINFOKEY + '.' + acctok['username'], reqbody.decode(), lockmd) log.info('msg="PutUserInfo" user="%s" filename="%s" fileid="%s" token="%s"' % (acctok['userid'][-20:], acctok['filename'], fileid, flask.request.args['access_token'][-20:])) return 'OK', http.client.OK From b0164ccbe3c8fa96b538b2335b42c7639e66e407 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 11 Jan 2023 14:34:01 +0100 Subject: [PATCH 149/325] Bridged apps: moved loadplugin to init, so that eventually we can drop the discovery stuff --- src/bridge/__init__.py | 32 +++++++++++++++++++++----------- src/wopiserver.py | 2 +- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index 83101a20..a20d7985 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -24,8 +24,8 @@ # The supported plugins integrated with the WOPI Bridge extensions BRIDGE_EXT_PLUGINS = {'md': 'codimd', 'txt': 'codimd', 'zmd': 'codimd', 'epd': 'etherpad', 'zep': 'etherpad'} -# The header that bridged apps are expected to send to the save endpoint -BRIDGED_APP_HEADER = 'X-Efss-Bridged-App' +# A header that bridged apps MUST send to the save endpoint to identify themselves +BRIDGED_APPNAME_HEADER = 'X-Efss-Bridged-App' # a standard message to be displayed by the app when some content might be lost: this would only # appear in case of uncaught exceptions or bugs handling the webhook callbacks @@ -72,6 +72,17 @@ def init(cls, config, log, secret): cls.hashsecret = secret cls.log = wopic.log = log wopic.sslverify = cls.sslverify + # now look for and load plugins for supported apps if configured + for app in BRIDGE_EXT_PLUGINS.values(): + url = config.get('general', f'{app}url', fallback=None) + if url: + inturl = config.get('general', f'{app}inturl', fallback=None) + try: + with open(f'/var/run/secrets/{app}_apikey', encoding='utf-8') as f: + apikey = f.readline().strip('\n') + except FileNotFoundError: + apikey = None + cls.loadplugin(app, url, inturl, apikey) @classmethod def loadplugin(cls, appname, appurl, appinturl, apikey): @@ -143,11 +154,16 @@ def appopen(wopisrc, acctok, appmd, viewmode, revatok=None): # TODO when using the wopiopen.py tool, the access token has to be decoded, to be clarified acctok = acctok.decode() - # load plugin if not done at init time and validate URLs + # (re)load plugin and validate URLs appname, appurl, appinturl, apikey = appmd try: WB.loadplugin(appname, appurl, appinturl, apikey) + appname = _validateappname(appname) + app = WB.plugins[appname] + WB.log.debug('msg="BridgeOpen: processing supported app" appname="%s" plugin="%s"' % (appname, app)) except ValueError: + WB.log.warning('msg="BridgeOpen: appname not supported or missing plugin" appname="%s" token="%s"' % + (appname, acctok[-20:])) raise FailedOpen('Failed to load WOPI bridge plugin for %s' % appname, http.client.INTERNAL_SERVER_ERROR) except KeyError: raise FailedOpen('Bridged app %s already configured with a different appurl' % appname, http.client.NOT_IMPLEMENTED) @@ -158,12 +174,6 @@ def appopen(wopisrc, acctok, appmd, viewmode, revatok=None): WB.log.warning('msg="BridgeOpen: unable to fetch file WOPI metadata" response="%d"' % res.status_code) raise FailedOpen('Invalid WOPI context', http.client.NOT_FOUND) filemd = res.json() - app = WB.plugins.get(appname.lower()) - if not app: - WB.log.warning('msg="BridgeOpen: appname not supported or missing plugin" filename="%s" appname="%s" token="%s"' % - (filemd['BaseFileName'], appname, acctok[-20:])) - raise FailedOpen('File type not supported', http.client.BAD_REQUEST) - WB.log.debug('msg="BridgeOpen: processing supported app" appname="%s" plugin="%s"' % (appname, app)) try: # use the 'UserCanWrite' attribute to decide whether the file is to be opened in read-only mode @@ -254,7 +264,7 @@ def appsave(docid): # ensure a save request comes from known/registered applications: # this is done via a specific header - appname = _validateappname(flask.request.headers[BRIDGED_APP_HEADER]) + appname = _validateappname(flask.request.headers[BRIDGED_APPNAME_HEADER]) WB.log.info('msg="BridgeSave: requested action" isclose="%s" docid="%s" app="%s" wopisrc="%s" token="%s"' % (isclose, docid, appname, wopisrc, acctok[-20:])) except KeyError as e: @@ -263,7 +273,7 @@ def appsave(docid): return wopic.jsonify('Missing metadata, could not save. %s' % RECOVER_MSG), http.client.BAD_REQUEST except ValueError: WB.log.error('msg="BridgeSave: unknown application" address="%s" appheader="%s" args="%s"' % - (flask.request.remote_addr, flask.request.headers.get(BRIDGED_APP_HEADER), flask.request.args)) + (flask.request.remote_addr, flask.request.headers.get(BRIDGED_APPNAME_HEADER), flask.request.args)) return wopic.jsonify('Unknown application, could not save. %s' % RECOVER_MSG), http.client.BAD_REQUEST # decide whether to notify the save thread diff --git a/src/wopiserver.py b/src/wopiserver.py index 8b9a876b..3570efb4 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -169,7 +169,7 @@ def init(cls): core.discovery.codetypes = cls.codetypes core.discovery.config = cls.config utils.endpoints = core.discovery.endpoints - except (configparser.NoOptionError, OSError) as e: + except (configparser.NoOptionError, OSError, ValueError) as e: # any error we get here with the configuration is fatal cls.log.fatal('msg="Failed to initialize the service, aborting" error="%s"' % e) print("Failed to initialize the service: %s\n" % e, file=sys.stderr) From 0e53b9e94799689e959400e5e232b2e7449d6129 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 26 Jan 2023 17:18:51 +0100 Subject: [PATCH 150/325] Fixed log formatting --- src/core/wopiutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 32cea911..a73951ca 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -238,10 +238,10 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app if 'MS 365' in appname: srv.allusers.add(userid) log.info('msg="Access token generated" userid="%s" wopiuser="%s" mode="%s" endpoint="%s" filename="%s" inode="%s" ' - 'mtime="%s" folderurl="%s" appname="%s" %s expiration="%d" token="%s"' % + 'mtime="%s" folderurl="%s" appname="%s"%s expiration="%d" token="%s"' % (userid[-20:], wopiuser if wopiuser != userid else username, viewmode, endpoint, statinfo['filepath'], statinfo['inode'], statinfo['mtime'], - folderurl, appname, 'forcelock="True"' if forcelock else '', exptime, acctok[-20:])) + folderurl, appname, ' forcelock="True"' if forcelock else '', exptime, acctok[-20:])) return statinfo['inode'], acctok, viewmode From 5ae05a878ef069008de65b5698a33f936c7006cc Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 30 Jan 2023 08:48:08 +0100 Subject: [PATCH 151/325] Added Javi --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1032608b..073a42a0 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Contributors (oldest contributions first): - Jörn Friedrich Dreyer (@butonic) - Michael Barz (@micbar) - Robert Kaussow (@xoxys) +- Javier Ferrer (@javfg) Initial revision: December 2016
First production version for CERNBox: September 2017 (presented at [oCCon17](https://occon17.owncloud.org) - [slides](https://www.slideshare.net/giuseppelopresti/collaborative-editing-and-more-in-cernbox))
From 1d314114b44679d02db2337f9eb45469ad0b73f9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 30 Jan 2023 08:58:17 +0100 Subject: [PATCH 152/325] Version 9.4 --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a33c44e3..5db27808 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ ## Changelog for the WOPI server +### Mon Jan 30 2023 - v9.4.0 +- Introduced support to forcefully evict valid locks + to compensate Microsoft Online mishandling of collaborative + sessions. This workaround will stay until a proper fix + is implemented following Microsoft CSPP team's advices +- Improved logging, in particular around lock eviction +- Bridged apps: moved plugin loading apps out of the deprecated + discovery module, and fixed some minor bugs +- CI: moved release builds to GitHub actions + ### Thu Nov 24 2022 - v9.3.0 - Introduced heuristic to log which sessions are allowed to open a collaborative session and which ones are From b3ec92dd933bba1d87e60a1e6a4197c9a52c49eb Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 31 Jan 2023 12:03:27 +0100 Subject: [PATCH 153/325] Shorten time to consider a session as resolved --- src/core/wopiutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index a73951ca..953e133a 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -143,8 +143,8 @@ def validateAndLogHeaders(op): # update bookkeeping of pending sessions if op.title() == 'Checkfileinfo' and session in srv.conflictsessions['pending'] and \ - int(srv.conflictsessions['pending'][session]['time']) < time.time() - 300: - # a previously conflicted session is still around executing Checkfileinfo after 5 minutes, assume it got resolved + int(srv.conflictsessions['pending'][session]['time']) < time.time() - 30: + # a previously conflicted session is still around executing Checkfileinfo after some time, assume it got resolved _resolveSession(session, acctok['filename']) return acctok, None From 5a937370eae6f6b9bb81fe5efeb459cec5470b03 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 31 Jan 2023 12:03:57 +0100 Subject: [PATCH 154/325] Retagging v9.4.0 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5db27808..647afae6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Changelog for the WOPI server -### Mon Jan 30 2023 - v9.4.0 +### Tue Jan 31 2023 - v9.4.0 - Introduced support to forcefully evict valid locks to compensate Microsoft Online mishandling of collaborative sessions. This workaround will stay until a proper fix From 7a07f9b9cb8e2283cb2f305a1654772673759969 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 31 Jan 2023 13:31:19 +0100 Subject: [PATCH 155/325] Improved logging on successful locks --- src/core/wopi.py | 6 +++--- src/core/wopiutils.py | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 34c56a33..52a14953 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -256,7 +256,7 @@ def setLock(fileid, reqheaders, acctok): log.warning('msg="Unable to set lastwritetime xattr" lockop="%s" user="%s" filename="%s" token="%s" reason="%s"' % (op.title(), acctok['userid'][-20:], fn, flask.request.args['access_token'][-20:], e)) - return utils.makeLockSuccessResponse(op, acctok, lock, 'v%s' % statInfo['etag']) + return utils.makeLockSuccessResponse(op, acctok, lock, oldLock, 'v%s' % statInfo['etag']) except IOError as e: if common.EXCL_ERROR in str(e): @@ -276,7 +276,7 @@ def setLock(fileid, reqheaders, acctok): evicted = utils.checkAndEvictLock(acctok['userid'], acctok['appname'], retrievedLock, oldLock, lock, acctok['endpoint'], fn, int(statInfo['mtime'])) if evicted: - return utils.makeLockSuccessResponse(op, acctok, lock, 'v%s' % statInfo['etag']) + return utils.makeLockSuccessResponse(op, acctok, lock, oldLock, 'v%s' % statInfo['etag']) else: return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, fn, 'The file is locked by %s' % @@ -287,7 +287,7 @@ def setLock(fileid, reqheaders, acctok): try: st.refreshlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(lock), utils.encodeLock(oldLock)) - return utils.makeLockSuccessResponse(op, acctok, lock, 'v%s' % statInfo['etag']) + return utils.makeLockSuccessResponse(op, acctok, lock, oldLock, 'v%s' % statInfo['etag']) except IOError as rle: # this is unexpected now log.error('msg="Failed to refresh lock" lockop="%s" filename="%s" token="%s" lock="%s" error="%s"' % diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 953e133a..44a0cfe4 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -454,15 +454,17 @@ def _resolveSession(session, filename): srv.openfiles[filename][1].add(session) -def makeLockSuccessResponse(operation, acctok, lock, version): +def makeLockSuccessResponse(operation, acctok, lock, oldlock, version): '''Generates and logs an HTTP 200 response with appropriate headers for Lock/RefreshLock operations''' session = flask.request.headers.get('X-WOPI-SessionId') if not session: session = acctok['username'] _resolveSession(session, acctok['filename']) - log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" lock="%s" version="%s"' % - (operation.title(), acctok['filename'], flask.request.args['access_token'][-20:], session, lock, version)) + log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" ' + 'lock="%s" oldlock="%s" version="%s"' % + (('UnlockAndRelock' if oldlock else operation.title()), acctok['filename'], + flask.request.args['access_token'][-20:], session, lock, oldlock, version)) resp = flask.Response() resp.status_code = http.client.OK resp.headers['X-WOPI-ItemVersion'] = version From 67273bf12848022b4afc3d7083c833e700937270 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 31 Jan 2023 13:40:41 +0100 Subject: [PATCH 156/325] Added file type to pending and resolved sessions status --- src/core/wopiutils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 44a0cfe4..0ac08a70 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -424,7 +424,8 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, filename srv.conflictsessions['pending'][session] = { 'user': user, 'time': int(time.time()), - 'heldby': retrievedlock + 'heldby': retrievedlock, + 'type': os.path.splitext(filename)[1], } if savetime: fileage = '%1.1f' % (time.time() - int(savetime)) @@ -445,7 +446,8 @@ def _resolveSession(session, filename): s = srv.conflictsessions['pending'].pop(session) srv.conflictsessions['resolved'][session] = { 'user': s['user'], - 'restime': int(time.time() - int(s['time'])) + 'restime': int(time.time() - int(s['time'])), + 'type': s['type'], } # keep some accounting of the open files if filename not in srv.openfiles: From b63baa36876e30abf8cc3dd69b10ecd4b3a8c120 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 31 Jan 2023 13:46:32 +0100 Subject: [PATCH 157/325] CI: fixed release action --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 74530b83..efcd9ac4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,4 +36,4 @@ jobs: context: . push: true file: ${{ matrix.file }} - tags: ${{ format('{0}/{1}', secrets.DOCKERHUB_ORGANIZATION, matrix.tag) }} + tags: ${{ format('{0}/{1}', vars.DOCKERHUB_ORGANIZATION, matrix.tag) }} From 4812b28e168be23179bf2ffd034d33e949ad0285 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 2 Feb 2023 16:58:56 +0100 Subject: [PATCH 158/325] CI: fixed version tagging Now the reva workflow is reused here --- .github/workflows/release.yml | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index efcd9ac4..27b69e04 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,34 +6,17 @@ on: jobs: release: - runs-on: self-hosted strategy: matrix: include: - - tag: wopiserver:${{ github.ref_name }} + - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }} file: wopiserver.Dockerfile - - tag: wopiserver:${{ github.ref_name }}-xrootd + - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-xrootd file: wopiserver-xrootd.Dockerfile - - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - if: ${{ github.event_name != 'pull_request' }} - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push ${{ matrix.tag }} - uses: docker/build-push-action@v3 - env: - VERSION: ${{ github.ref_name }} - with: - context: . - push: true - file: ${{ matrix.file }} - tags: ${{ format('{0}/{1}', vars.DOCKERHUB_ORGANIZATION, matrix.tag) }} + uses: cs3org/reva/.github/workflows/docker.yml@master + secrets: inherit + with: + file: ${{ matrix.file }} + tags: ${{ matrix.tags }} + build-args: VERSION=${{ github.ref_name }} + push: true From bf0f76d04955b5ed473206e6ef0a0802c0e2d3c7 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 2 Feb 2023 17:04:15 +0100 Subject: [PATCH 159/325] CI: also tag the `latest` docker image --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 27b69e04..427d0eaf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: include: - - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }} + - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }},${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:latest file: wopiserver.Dockerfile - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-xrootd file: wopiserver-xrootd.Dockerfile From a32a2f5e35759fe80def216494b5bcdb95d56219 Mon Sep 17 00:00:00 2001 From: Tomas Zaluckij Date: Sat, 4 Feb 2023 20:13:20 +0000 Subject: [PATCH 160/325] CI: build an additional `arm64` docker image --- .github/workflows/release.yml | 14 ++++++++++++++ wopiserver-arm64.Dockerfile | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 wopiserver-arm64.Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 427d0eaf..c5939b67 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,3 +20,17 @@ jobs: tags: ${{ matrix.tags }} build-args: VERSION=${{ github.ref_name }} push: true + release-arm64: + strategy: + matrix: + include: + - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-arm64 + file: wopiserver-arm64.Dockerfile + uses: cs3org/reva/.github/workflows/docker.yml@master + secrets: inherit + with: + file: ${{ matrix.file }} + tags: ${{ matrix.tags }} + build-args: VERSION=${{ github.ref_name }} + push: true + platforms: linux/arm64 diff --git a/wopiserver-arm64.Dockerfile b/wopiserver-arm64.Dockerfile new file mode 100644 index 00000000..562dd99a --- /dev/null +++ b/wopiserver-arm64.Dockerfile @@ -0,0 +1,34 @@ +# Dockerfile for WOPI Server +# +# Build: WOPI_DOCKER_TYPE=-arm64 docker-compose -f wopiserver.yaml build --build-arg VERSION=`git describe | sed 's/^v//'` wopiserver +# Run: docker-compose -f wopiserver.yaml up -d + +FROM python:3.10-slim-buster + +ARG VERSION=latest + +LABEL maintainer="cernbox-admins@cern.ch" \ + org.opencontainers.image.title="The ScienceMesh IOP WOPI server" \ + org.opencontainers.image.version="$VERSION" + +# prerequisites: we explicitly install g++ as it is required by grpcio but missing from its dependencies +WORKDIR /app +COPY requirements.txt . +RUN apt -y install g++ && \ + pip3 install --upgrade pip setuptools && \ + pip3 install --no-cache-dir --upgrade -r requirements.txt + +# install software +RUN mkdir -p /app/core /app/bridge /test /etc/wopi /var/log/wopi /var/wopi_local_storage +COPY ./src/* ./tools/* /app/ +COPY ./src/core/* /app/core/ +COPY ./src/bridge/* /app/bridge/ +RUN sed -i "s/WOPISERVERVERSION = 'git'/WOPISERVERVERSION = '$VERSION'/" /app/wopiserver.py && \ + grep 'WOPISERVERVERSION =' /app/wopiserver.py +COPY wopiserver.conf /etc/wopi/wopiserver.defaults.conf +COPY test/*py test/*conf /test/ + +# add basic custom configuration; need to contextualize +COPY ./docker/etc/*secret ./docker/etc/wopiserver.conf /etc/wopi/ + +ENTRYPOINT ["/app/wopiserver.py"] From 34ffa99ddfe231a19cd9f5052b74c903d2220002 Mon Sep 17 00:00:00 2001 From: Vasco Guita <33404234+vascoguita@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:46:13 +0100 Subject: [PATCH 161/325] Use python:3.10-slim-buster for arm64 and amd64 docker images --- .github/workflows/release.yml | 18 +++--------------- wopiserver-arm64.Dockerfile | 34 ---------------------------------- wopiserver.Dockerfile | 2 +- 3 files changed, 4 insertions(+), 50 deletions(-) delete mode 100644 wopiserver-arm64.Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5939b67..58e4a0f9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,7 @@ on: push: tags: - "*" + workflow_dispatch: jobs: release: @@ -19,18 +20,5 @@ jobs: file: ${{ matrix.file }} tags: ${{ matrix.tags }} build-args: VERSION=${{ github.ref_name }} - push: true - release-arm64: - strategy: - matrix: - include: - - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-arm64 - file: wopiserver-arm64.Dockerfile - uses: cs3org/reva/.github/workflows/docker.yml@master - secrets: inherit - with: - file: ${{ matrix.file }} - tags: ${{ matrix.tags }} - build-args: VERSION=${{ github.ref_name }} - push: true - platforms: linux/arm64 + push: ${{ github.event_name != 'workflow_dispatch' }} + platforms: linux/amd64,linux/arm64 diff --git a/wopiserver-arm64.Dockerfile b/wopiserver-arm64.Dockerfile deleted file mode 100644 index 562dd99a..00000000 --- a/wopiserver-arm64.Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# Dockerfile for WOPI Server -# -# Build: WOPI_DOCKER_TYPE=-arm64 docker-compose -f wopiserver.yaml build --build-arg VERSION=`git describe | sed 's/^v//'` wopiserver -# Run: docker-compose -f wopiserver.yaml up -d - -FROM python:3.10-slim-buster - -ARG VERSION=latest - -LABEL maintainer="cernbox-admins@cern.ch" \ - org.opencontainers.image.title="The ScienceMesh IOP WOPI server" \ - org.opencontainers.image.version="$VERSION" - -# prerequisites: we explicitly install g++ as it is required by grpcio but missing from its dependencies -WORKDIR /app -COPY requirements.txt . -RUN apt -y install g++ && \ - pip3 install --upgrade pip setuptools && \ - pip3 install --no-cache-dir --upgrade -r requirements.txt - -# install software -RUN mkdir -p /app/core /app/bridge /test /etc/wopi /var/log/wopi /var/wopi_local_storage -COPY ./src/* ./tools/* /app/ -COPY ./src/core/* /app/core/ -COPY ./src/bridge/* /app/bridge/ -RUN sed -i "s/WOPISERVERVERSION = 'git'/WOPISERVERVERSION = '$VERSION'/" /app/wopiserver.py && \ - grep 'WOPISERVERVERSION =' /app/wopiserver.py -COPY wopiserver.conf /etc/wopi/wopiserver.defaults.conf -COPY test/*py test/*conf /test/ - -# add basic custom configuration; need to contextualize -COPY ./docker/etc/*secret ./docker/etc/wopiserver.conf /etc/wopi/ - -ENTRYPOINT ["/app/wopiserver.py"] diff --git a/wopiserver.Dockerfile b/wopiserver.Dockerfile index f72cb164..2a1018b4 100644 --- a/wopiserver.Dockerfile +++ b/wopiserver.Dockerfile @@ -2,7 +2,7 @@ # # Build: make docker or docker-compose -f wopiserver.yaml build --build-arg VERSION=`git describe | sed 's/^v//'` wopiserver -FROM python:3.11-alpine +FROM python:3.10-slim-buster ARG VERSION=latest From 3b67b801aaf6ebaab44281bb68739916e965305e Mon Sep 17 00:00:00 2001 From: Tomas Zaluckij Date: Tue, 7 Feb 2023 16:32:22 +0000 Subject: [PATCH 162/325] Use correct package manager (apt) for the slim-buster image --- wopiserver.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wopiserver.Dockerfile b/wopiserver.Dockerfile index 2a1018b4..1c889f67 100644 --- a/wopiserver.Dockerfile +++ b/wopiserver.Dockerfile @@ -13,7 +13,7 @@ LABEL maintainer="cernbox-admins@cern.ch" \ # prerequisites: we explicitly install g++ as it is required by grpcio but missing from its dependencies WORKDIR /app COPY requirements.txt . -RUN apk add g++ && \ +RUN apt -y install g++ && \ pip3 install --upgrade pip setuptools && \ pip3 install --no-cache-dir --upgrade -r requirements.txt From 49fae003efe66a98deb78910247d965c318244d3 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 8 Feb 2023 17:26:30 +0100 Subject: [PATCH 163/325] Reworked to support different images on different platforms --- .github/workflows/release.yml | 19 +++++++++++++------ wopiserver.Dockerfile | 15 ++++++++++----- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 58e4a0f9..b26f507c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,15 +10,22 @@ jobs: strategy: matrix: include: - - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }},${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:latest - file: wopiserver.Dockerfile - - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-xrootd - file: wopiserver-xrootd.Dockerfile + - file: wopiserver.Dockerfile + tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }},${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:latest + platform: linux/amd64 + image: python:3.11-alpine + - file: wopiserver.Dockerfile + tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }},${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:latest + platform: linux/arm64 + image: python:3.10-slim-buster + - file: wopiserver-xrootd.Dockerfile + tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-xrootd + platform: linux/amd64 uses: cs3org/reva/.github/workflows/docker.yml@master secrets: inherit with: file: ${{ matrix.file }} tags: ${{ matrix.tags }} - build-args: VERSION=${{ github.ref_name }} + build-args: VERSION=${{ github.ref_name }},BASEIMAGE=${{ matrix.image }} push: ${{ github.event_name != 'workflow_dispatch' }} - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.platform }} diff --git a/wopiserver.Dockerfile b/wopiserver.Dockerfile index 1c889f67..c419cc01 100644 --- a/wopiserver.Dockerfile +++ b/wopiserver.Dockerfile @@ -1,10 +1,11 @@ # Dockerfile for WOPI Server # -# Build: make docker or docker-compose -f wopiserver.yaml build --build-arg VERSION=`git describe | sed 's/^v//'` wopiserver - -FROM python:3.10-slim-buster +# Build: make docker or docker-compose -f wopiserver.yaml build --build-arg VERSION=`git describe | sed 's/^v//'` BASEIMAGE=... wopiserver ARG VERSION=latest +ARG BASEIMAGE=python:3.11-alpine + +FROM $BASEIMAGE LABEL maintainer="cernbox-admins@cern.ch" \ org.opencontainers.image.title="The ScienceMesh IOP WOPI server" \ @@ -13,8 +14,12 @@ LABEL maintainer="cernbox-admins@cern.ch" \ # prerequisites: we explicitly install g++ as it is required by grpcio but missing from its dependencies WORKDIR /app COPY requirements.txt . -RUN apt -y install g++ && \ - pip3 install --upgrade pip setuptools && \ +RUN if [ '$BASEIMAGE' = 'python:3.10-slim-buster' ]; then \ + apt -y install g++; \ + else \ + apk add g++; \ + fi +RUN pip3 install --upgrade pip setuptools && \ pip3 install --no-cache-dir --upgrade -r requirements.txt # install software From a8ee77be6cc1a12bcb5474a069c5dd62078333ac Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 10 Feb 2023 10:46:16 +0100 Subject: [PATCH 164/325] CI: fixed arm64 build Replaced reusable action with a hard-coded clone so that we can pass multiple build arguments to docker --- .github/workflows/release.yml | 47 +++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b26f507c..6fd2ecc0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,43 @@ on: workflow_dispatch: jobs: +# The following is a clone of cs3org/reva/.github/workflows/docker.yml because reusable actions do not (yet) support lists as input types: +# see https://github.com/community/community/discussions/11692 release: + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v3.1.0 + - name: Set up QEMU + if: matrix.platforms != '' + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + if: matrix.push + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build ${{ matrix.push && 'and push' || '' }} ${{ matrix.tags }} Docker image + uses: docker/build-push-action@v3 + with: + context: . + file: ${{ matrix.file }} + tags: ${{ matrix.tags }} + push: ${{ matrix.push }} + build-args: | + VERSION=${{ github.ref_name }} + BASEIMAGE=${{ matrix.image }} + platforms: ${{ matrix.platforms }} +# - name: Upload ${{ matrix.tags }} Docker image to artifacts +# uses: ishworkh/docker-image-artifact-upload@v1 +# if: ${{ inputs.load }} +# with: +# image: ${{ inputs.tags }} +# retention_days: '1' +# end of the clone + strategy: matrix: include: @@ -14,18 +50,13 @@ jobs: tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }},${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:latest platform: linux/amd64 image: python:3.11-alpine + push: ${{ github.event_name != 'workflow_dispatch' }} - file: wopiserver.Dockerfile tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }},${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:latest platform: linux/arm64 image: python:3.10-slim-buster + push: ${{ github.event_name != 'workflow_dispatch' }} - file: wopiserver-xrootd.Dockerfile tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-xrootd platform: linux/amd64 - uses: cs3org/reva/.github/workflows/docker.yml@master - secrets: inherit - with: - file: ${{ matrix.file }} - tags: ${{ matrix.tags }} - build-args: VERSION=${{ github.ref_name }},BASEIMAGE=${{ matrix.image }} - push: ${{ github.event_name != 'workflow_dispatch' }} - platforms: ${{ matrix.platform }} + push: ${{ github.event_name != 'workflow_dispatch' }} From 1bf378a077a61cabf8cb052ac356531aed69e8b4 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 10 Feb 2023 10:52:34 +0100 Subject: [PATCH 165/325] CI: disable fail-fast --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6fd2ecc0..2ab80424 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,7 @@ jobs: # end of the clone strategy: + fail-fast: false matrix: include: - file: wopiserver.Dockerfile From 95d687ff6ade6041132dcd53fd9abea3ed5ff9a2 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sun, 19 Feb 2023 10:25:39 +0100 Subject: [PATCH 166/325] CI: better logic to choose package manager --- wopiserver.Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wopiserver.Dockerfile b/wopiserver.Dockerfile index c419cc01..1157dce0 100644 --- a/wopiserver.Dockerfile +++ b/wopiserver.Dockerfile @@ -14,10 +14,12 @@ LABEL maintainer="cernbox-admins@cern.ch" \ # prerequisites: we explicitly install g++ as it is required by grpcio but missing from its dependencies WORKDIR /app COPY requirements.txt . -RUN if [ '$BASEIMAGE' = 'python:3.10-slim-buster' ]; then \ - apt -y install g++; \ +RUN if command -v apt &> /dev/null; then \ + echo "Using apt"; apt -y install g++; \ + elif command -v apk &> /dev/null; then \ + echo "Using apk"; apk add g++; \ else \ - apk add g++; \ + echo "This distribution does not provide a supported package manager"; false; \ fi RUN pip3 install --upgrade pip setuptools && \ pip3 install --no-cache-dir --upgrade -r requirements.txt From 44632d5eb129c3803fa19af3c150e31b3190f1c9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 21 Feb 2023 09:00:09 +0100 Subject: [PATCH 167/325] Use the single-file frontend view when available as a breadcrumb folder URL --- src/core/wopi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 52a14953..4a266b8a 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -57,7 +57,7 @@ def checkFileInfo(fileid, acctok): fmd['FileSharingPostMessage'] = True furl = acctok['folderurl'] if furl != '/': - fmd['BreadcrumbFolderUrl'] = furl + fmd['BreadcrumbFolderUrl'] = furl + '?scrollTo=' + fmd['BaseFileName'] if acctok['username'] == '': fmd['IsAnonymousUser'] = True fmd['UserFriendlyName'] = 'Guest ' + utils.randomString(3) From bdd3f8c7ede56655158f289bc0fdb4d35b2a984a Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 21 Feb 2023 09:03:45 +0100 Subject: [PATCH 168/325] Updated contributors --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 073a42a0..3c4fe9a1 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ Contributors (oldest contributions first): - Michael Barz (@micbar) - Robert Kaussow (@xoxys) - Javier Ferrer (@javfg) + Vasco Guita (@vascoguita) + Tomas Zaluckij (@Tomaszal) Initial revision: December 2016
First production version for CERNBox: September 2017 (presented at [oCCon17](https://occon17.owncloud.org) - [slides](https://www.slideshare.net/giuseppelopresti/collaborative-editing-and-more-in-cernbox))
From 6e3fc660b5a46a0a051eba12cc97e907e6fe7a55 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 21 Feb 2023 12:24:40 +0100 Subject: [PATCH 169/325] Log direct download action --- src/core/wopi.py | 4 ++-- src/wopiserver.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 4a266b8a..3a09d037 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -133,8 +133,8 @@ def getFile(_fileid, acctok): f = peekable(st.readfile(acctok['endpoint'], acctok['filename'], acctok['userid'], None)) firstchunk = f.peek() if isinstance(firstchunk, IOError): - log.error('msg="GetFile: download failed" filename="%s" token="%s" error="%s"' % - (acctok['filename'], flask.request.args['access_token'][-20:], firstchunk)) + log.error('msg="GetFile: download failed" endpoint="%s" filename="%s" token="%s" error="%s"' % + (acctok['endpoint'], acctok['filename'], flask.request.args['access_token'][-20:], firstchunk)) return 'Failed to fetch file from storage', http.client.INTERNAL_SERVER_ERROR # stat the file to get the current version statInfo = st.statx(acctok['endpoint'], acctok['filename'], acctok['userid']) diff --git a/src/wopiserver.py b/src/wopiserver.py index 3570efb4..edbae292 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -373,6 +373,9 @@ def iopDownload(): acctok = jwt.decode(flask.request.args['access_token'], Wopi.wopisecret, algorithms=['HS256']) if acctok['exp'] < time.time(): raise jwt.exceptions.ExpiredSignatureError + Wopi.log.info('msg="iopDownload: returning contents" client="%s" endpoint="%s" filename="%s" token="%s"' % + (flask.request.remote_addr, acctok['endpoint'], acctok['filename'], + flask.request.args['access_token'][-20:])) return core.wopi.getFile(0, acctok) # note that here we exploit the non-dependency from fileid except (jwt.exceptions.DecodeError, jwt.exceptions.ExpiredSignatureError, KeyError) as e: Wopi.log.info('msg="Expired or malformed token" client="%s" requestedUrl="%s" error="%s" token="%s"' % From c8cd2dace48cb401d80323036af9109893782c56 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 21 Feb 2023 12:39:47 +0100 Subject: [PATCH 170/325] xroot: improved logging --- src/core/xrootiface.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index bdf0f21c..9bd6b6aa 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -123,8 +123,12 @@ def _xrootcmd(endpoint, cmd, subcmd, userid, args, app='wopi'): # failure: get info from stderr, log and raise msg = res[1][res[1].find('=') + 1:].strip('\n') if common.ENOENT_MSG.lower() in msg or 'unable to get attribute' in msg or rc == '2': - log.info('msg="Invoked cmd on non-existing entity" cmd="%s" subcmd="%s" args="%s" result="%s" rc="%s"' % - (cmd, subcmd, args, msg.replace('error:', '').strip(), rc.strip('\00'))) + if 'attribute' in msg: + log.debug('msg="Missing attribute on file" cmd="%s" subcmd="%s" args="%s" result="%s" rc="%s"' % + (cmd, subcmd, args, msg.replace('error:', '').strip(), rc.strip('\00'))) + else: + log.info('msg="File not found" url="%s" cmd="%s" args="%s" result="%s" rc="%s"' % + (_geturlfor(endpoint), cmd, args, msg.replace('error:', '').strip(), rc.strip('\00'))) raise IOError(common.ENOENT_MSG) if EXCL_XATTR_MSG in msg: log.info('msg="Invoked setxattr on an already locked entity" args="%s" result="%s" rc="%s"' % From d2a6a84bcc12b2f18b92bd89c42c5f75985b9120 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 21 Feb 2023 12:50:19 +0100 Subject: [PATCH 171/325] Use proper username instead of display name where necessary This includes PutUserInfo's xattr key as well as the storeForRecovery logic. The username is now kept all along also when using xroot. Also, the configuration was amended to indicate `homepath`, with `conflictpath` as deprecated key. --- src/core/wopi.py | 8 ++++---- src/core/wopiutils.py | 24 ++++++++++++------------ src/core/xrootiface.py | 4 ++-- src/wopiserver.py | 7 ++++--- wopiserver.conf | 5 +++-- 5 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 3a09d037..921921c1 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -95,7 +95,7 @@ def checkFileInfo(fileid, acctok): fmd['SupportsContainers'] = False # TODO this is all to be implemented fmd['SupportsUserInfo'] = True uinfo = st.getxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], - utils.USERINFOKEY + '.' + acctok['username']) + utils.USERINFOKEY + '.' + acctok['wopiuser'].split('@')[0]) if uinfo: fmd['UserInfo'] = uinfo @@ -544,7 +544,7 @@ def _createNewFile(fileid, acctok): (acctok['userid'][-20:], acctok['filename'], flask.request.args['access_token'][-20:])) return 'OK', http.client.OK except IOError as e: - utils.storeForRecovery(flask.request.get_data(), acctok['username'], acctok['filename'], + utils.storeForRecovery(flask.request.get_data(), acctok['wopiuser'], acctok['filename'], flask.request.args['access_token'][-20:], e) return IO_ERROR, http.client.INTERNAL_SERVER_ERROR @@ -588,7 +588,7 @@ def putFile(fileid, acctok): return resp except IOError as e: - utils.storeForRecovery(flask.request.get_data(), acctok['username'], acctok['filename'], + utils.storeForRecovery(flask.request.get_data(), acctok['wopiuser'], acctok['filename'], flask.request.args['access_token'][-20:], e) return IO_ERROR, http.client.INTERNAL_SERVER_ERROR @@ -606,7 +606,7 @@ def putUserInfo(fileid, reqbody, acctok): lockmd = st.getlock(acctok['endpoint'], acctok['filename'], acctok['userid']) lockmd = (acctok['appname'], utils.encodeLock(lockmd)) if lockmd else None st.setxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], - utils.USERINFOKEY + '.' + acctok['username'], reqbody.decode(), lockmd) + utils.USERINFOKEY + '.' + acctok['wopiuser'].split('@')[0], reqbody.decode(), lockmd) log.info('msg="PutUserInfo" user="%s" filename="%s" fileid="%s" token="%s"' % (acctok['userid'][-20:], acctok['filename'], fileid, flask.request.args['access_token'][-20:])) return 'OK', http.client.OK diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 0ac08a70..64ad9b84 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -237,10 +237,9 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app acctok = jwt.encode(tokmd, srv.wopisecret, algorithm='HS256') if 'MS 365' in appname: srv.allusers.add(userid) - log.info('msg="Access token generated" userid="%s" wopiuser="%s" mode="%s" endpoint="%s" filename="%s" inode="%s" ' - 'mtime="%s" folderurl="%s" appname="%s"%s expiration="%d" token="%s"' % - (userid[-20:], wopiuser if wopiuser != userid else username, viewmode, endpoint, - statinfo['filepath'], statinfo['inode'], statinfo['mtime'], + log.info('msg="Access token generated" userid="%s" wopiuser="%s" username="%s" mode="%s" endpoint="%s" filename="%s" ' + 'inode="%s" mtime="%s" folderurl="%s" appname="%s"%s expiration="%d" token="%s"' % + (userid[-20:], wopiuser, username, viewmode, endpoint, statinfo['filepath'], statinfo['inode'], statinfo['mtime'], folderurl, appname, ' forcelock="True"' if forcelock else '', exptime, acctok[-20:])) return statinfo['inode'], acctok, viewmode @@ -460,7 +459,7 @@ def makeLockSuccessResponse(operation, acctok, lock, oldlock, version): '''Generates and logs an HTTP 200 response with appropriate headers for Lock/RefreshLock operations''' session = flask.request.headers.get('X-WOPI-SessionId') if not session: - session = acctok['username'] + session = acctok['wopiuser'] _resolveSession(session, acctok['filename']) log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" ' @@ -480,7 +479,7 @@ def storeWopiFile(acctok, retrievedlock, xakey, targetname=''): targetname = acctok['filename'] session = flask.request.headers.get('X-WOPI-SessionId') if not session: - session = acctok['username'] + session = acctok['wopiuser'] _resolveSession(session, targetname) writeerror = None @@ -513,8 +512,9 @@ def storeAfterConflict(acctok, retrievedlock, lock, reason): if common.ACCESS_ERROR not in str(e): dorecovery = e else: - # let's try the configured conflictpath instead of the current folder - newname = srv.conflictpath.replace('user_initial', acctok['username'][0]).replace('username', acctok['username']) \ + # let's try the configured user's (or owner's) homepath instead of the current folder + newname = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ + replace('username', acctok['wopiuser'].split('@')[0]) \ + os.path.sep + os.path.basename(newname) try: storeWopiFile(acctok, retrievedlock, LASTSAVETIMEKEY, newname) @@ -523,7 +523,7 @@ def storeAfterConflict(acctok, retrievedlock, lock, reason): dorecovery = e if dorecovery: - storeForRecovery(flask.request.get_data(), acctok['username'], newname, + storeForRecovery(flask.request.get_data(), acctok['wopiuser'], newname, flask.request.args['access_token'][-20:], dorecovery) # conflict file was stored on recovery space, tell user (but reason is advisory...) reason += ', please contact support to recover it' @@ -536,10 +536,10 @@ def storeAfterConflict(acctok, retrievedlock, lock, reason): acctok['filename'], reason) -def storeForRecovery(content, username, filename, acctokforlog, exception): +def storeForRecovery(content, wopiuser, filename, acctokforlog, exception): try: - filepath = srv.recoverypath + os.sep + time.strftime('%Y%m%dT%H%M%S') + '_editedby_' + secure_filename(username) \ - + '_origat_' + secure_filename(filename) + filepath = srv.recoverypath + os.sep + time.strftime('%Y%m%dT%H%M%S') + '_editedby_' \ + + secure_filename(wopiuser.split('@')[0]) + '_origat_' + secure_filename(filename) with open(filepath, mode='wb') as f: written = f.write(content) if written != len(content): diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 9bd6b6aa..477c6a30 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -86,10 +86,10 @@ def _geneoslock(appname): def _eosargs(userid, app='wopi', bookingsize=0): - '''Assume userid is in the form uid:gid and split it into uid, gid + '''Assume userid is in the form username@idp:uid:gid and split it into uid, gid plus generate extra EOS-specific arguments for the xroot URL''' try: - # try to assert that userid must follow a '%d:%d' format + # try to assert that userid must follow a '%s:%d:%d' format userid = userid.split(':') if len(userid) != 2: raise ValueError diff --git a/src/wopiserver.py b/src/wopiserver.py index edbae292..cdb796ab 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -141,7 +141,8 @@ def init(cls): cls.log.error('msg="Failed to open the provided certificate or key to start in https mode"') raise cls.wopiurl = cls.config.get('general', 'wopiurl') - cls.conflictpath = cls.config.get('general', 'conflictpath', fallback='/') + cls.homepath = cls.config.get('general', 'homepath', + fallback=cls.config.get('general', 'conflictpath', fallback='/home/username')) cls.recoverypath = cls.config.get('io', 'recoverypath', fallback='/var/spool/wopirecovery') try: os.makedirs(cls.recoverypath) @@ -332,8 +333,8 @@ def iopOpenInApp(): userid = storage.getuseridfromcreds(usertoken, wopiuser) if userid != usertoken: # this happens in hybrid deployments with xrootd as storage interface: - # in this case we override the wopiuser with the resolved uid:gid - wopiuser = userid + # in this case we decorate the wopiuser with the resolved uid:gid + wopiuser += ':' + userid inode, acctok, vm = utils.generateAccessToken(userid, fileid, viewmode, (username, wopiuser), folderurl, endpoint, (appname, appurl, appviewurl), forcelock=forcelock) except IOError as e: diff --git a/wopiserver.conf b/wopiserver.conf index 6582560e..9b77b8c3 100644 --- a/wopiserver.conf +++ b/wopiserver.conf @@ -109,14 +109,15 @@ wopilockexpiration = 1800 # This feature can be disabled in order to operate a pure WOPI server for online apps. #detectexternallocks = True -# Location of the webconflict files. By default, such files are stored in the same path +# Location of the user's personal space, used as a fall back location when storing +# webconflict files. By default, such files are stored in the same path # as the original file. If that fails (e.g. because of missing permissions), # an attempt is made to store such files in this path if specified, otherwise # the system falls back to the recovery space (cf. io|recoverypath). # The keywords and are replaced with the actual username's # initial letter and the actual username, respectively, so you can use e.g. # /your_storage/home/user_initial/username -#conflictpath = / +#homepath = /home/username # Disable write ability (i.e. force read-only) when an open is requested for an ODF # file with a Microsoft Office app. This allows to use MS Office as a pure viewer, From ac1f0de459e9e743545e6f4f87b81ef4b72c067e Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 23 Feb 2023 14:29:20 +0100 Subject: [PATCH 172/325] Added curl to docker images for health checks and used apk preferentially --- wopiserver.Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wopiserver.Dockerfile b/wopiserver.Dockerfile index 1157dce0..b0280e91 100644 --- a/wopiserver.Dockerfile +++ b/wopiserver.Dockerfile @@ -14,10 +14,10 @@ LABEL maintainer="cernbox-admins@cern.ch" \ # prerequisites: we explicitly install g++ as it is required by grpcio but missing from its dependencies WORKDIR /app COPY requirements.txt . -RUN if command -v apt &> /dev/null; then \ +RUN if command -v apk &> /dev/null; then \ + echo "Using apk"; apk add curl g++; \ + elif command -v apt &> /dev/null; then \ echo "Using apt"; apt -y install g++; \ - elif command -v apk &> /dev/null; then \ - echo "Using apk"; apk add g++; \ else \ echo "This distribution does not provide a supported package manager"; false; \ fi From 6935763f87cecb4a9e911b9ce92e3d5e5f2f7d53 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 23 Feb 2023 00:02:55 +0100 Subject: [PATCH 173/325] Improved PutRelative when user cannot save in the same folder --- src/core/wopi.py | 49 +++++++++++++++++++++++++++++-------------- src/core/wopiutils.py | 17 ++++++++++++++- src/wopiserver.py | 19 ++++++++--------- wopiserver.conf | 7 ++++--- 4 files changed, 62 insertions(+), 30 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 921921c1..ff1c895e 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -85,12 +85,11 @@ def checkFileInfo(fileid, acctok): fmd['SupportsExtendedLockLength'] = fmd['SupportsGetLock'] = True fmd['SupportsUpdate'] = fmd['UserCanWrite'] = fmd['SupportsLocks'] = \ fmd['SupportsDeleteFile'] = acctok['viewmode'] == utils.ViewMode.READ_WRITE - # SaveAs functionality is enabled only for owners and in READ_WRITE mode. Anonymous and federated users - # are never owners despite their wopiuser credentials may match the owner's. Federated/external accounts - # are given by Reva with their network domain in parenthesis, that's why we match them. - notOwner = fmd['OwnerId'] != fmd['UserId'] or fmd['IsAnonymousUser'] or \ - ('(' in acctok['username'] and acctok['username'][-1] == ')') - fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE or notOwner + # SaveAs functionality is disabled for anonymous and federated users when in read-only mode, as they have + # no personal space where to save as an alternate location. + # Note that single-file r/w shares are optimistically offered a SaveAs option, which may only work for primary users. + fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE \ + and not utils.isPrimaryUser(acctok) fmd['SupportsRename'] = fmd['UserCanRename'] = enablerename and (acctok['viewmode'] == utils.ViewMode.READ_WRITE) fmd['SupportsContainers'] = False # TODO this is all to be implemented fmd['SupportsUserInfo'] = True @@ -366,13 +365,19 @@ def putRelative(fileid, reqheaders, acctok): # either one xor the other MUST be present; note we can't use `^` as we have a mix of str and NoneType if (suggTarget and relTarget) or (not suggTarget and not relTarget): return 'Conflicting headers given', http.client.BAD_REQUEST + if acctok['viewmode'] != utils.ViewMode.READ_WRITE: + # here we must have an authenticated user with no write rights on the current folder: go to the user's homepath + targetName = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ + replace('username', acctok['wopiuser'].split('@')[0]) + os.path.sep + else: + targetName = os.path.dirname(acctok['filename']) if suggTarget: # the suggested target is a UTF7-encoded (!) filename that can be changed to avoid collisions suggTarget = suggTarget.encode().decode('utf-7') if suggTarget[0] == '.': # we just have the extension here - targetName = os.path.splitext(acctok['filename'])[0] + suggTarget + targetName += os.path.basename(os.path.splitext(acctok['filename'])[0]) + suggTarget else: - targetName = os.path.dirname(acctok['filename']) + os.path.sep + suggTarget + targetName += os.path.sep + suggTarget # check for existence of the target file and adjust until a non-existing one is obtained while True: try: @@ -391,7 +396,7 @@ def putRelative(fileid, reqheaders, acctok): return 'Error with the given target', http.client.INTERNAL_SERVER_ERROR else: # the relative target is a UTF7-encoded filename to be respected, and that may overwrite an existing file - relTarget = os.path.dirname(acctok['filename']) + os.path.sep + relTarget.encode().decode('utf-7') # make full path + relTarget = targetName + os.path.sep + relTarget.encode().decode('utf-7') # make full path try: # check for file existence statInfo = st.statx(acctok['endpoint'], relTarget, acctok['userid']) @@ -411,18 +416,29 @@ def putRelative(fileid, reqheaders, acctok): # optimistically assume we're clear pass targetName = relTarget + # either way, we now have a targetName to save the file: attempt to do so try: utils.storeWopiFile(acctok, None, utils.LASTSAVETIMEKEY, targetName) - newstat = st.statx(acctok['endpoint'], targetName, acctok['userid']) - _, newfileid = common.decodeinode(newstat['inode']) except IOError as e: - if str(e) == common.ACCESS_ERROR: - # BAD_REQUEST may seem better but the WOPI validator tests explicitly expect NOT_IMPLEMENTED + if str(e) != common.ACCESS_ERROR: + return IO_ERROR, http.client.INTERNAL_SERVER_ERROR + raisenoaccess = True + # make an attempt in the user's home if possible + if utils.isPrimaryUser(acctok): + targetName = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ + replace('username', acctok['wopiuser'].split('@')[0]) \ + + os.path.sep + os.path.basename(targetName) + try: + utils.storeWopiFile(acctok, None, utils.LASTSAVETIMEKEY, targetName) + raisenoaccess = False + except IOError: + # at this point give up and return error + pass + if raisenoaccess: + # UNAUTHORIZED may seem better but the WOPI validator tests explicitly expect NOT_IMPLEMENTED return 'Unauthorized to perform PutRelative', http.client.NOT_IMPLEMENTED - utils.storeForRecovery(flask.request.get_data(), acctok['username'], targetName, - flask.request.args['access_token'][-20:], e) - return IO_ERROR, http.client.INTERNAL_SERVER_ERROR + # generate an access token for the new file log.info('msg="PutRelative: generating new access token" user="%s" filename="%s" ' 'mode="ViewMode.READ_WRITE" friendlyname="%s"' % @@ -432,6 +448,7 @@ def putRelative(fileid, reqheaders, acctok): acctok['folderurl'], acctok['endpoint'], (acctok['appname'], acctok['appediturl'], acctok['appviewurl'])) # prepare and send the response as JSON + _, newfileid = common.decodeinode(inode) mdforhosturls = { 'appname': acctok['appname'], 'filename': targetName, diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 64ad9b84..4d86713e 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -17,7 +17,7 @@ from datetime import datetime from base64 import b64encode, b64decode from binascii import Error as B64Error -from urllib.parse import quote_plus as url_quote_plus +from urllib.parse import quote_plus as url_quote_plus, urlparse import http.client import flask import jwt @@ -33,6 +33,9 @@ # header used by reverse proxies such as traefik to pass the real remote IP address REALIPHEADER = 'X-Real-IP' +# the prefix used for public links, cf. reva:pkg/app/provider/wopi/wopi.go +PUBLINKPREFIX = '/files/link/public' + # convenience references to global entities st = None srv = None @@ -192,6 +195,18 @@ def randomString(size): return ''.join([choice(ascii_lowercase) for _ in range(size)]) +def isPrimaryUser(acctok): + '''Return whether a session belongs to a primary/internal account, in the context of + saving a webconflicted file or a SaveAs operation. + False means the user is either anonymous or federated/external, which is given by Reva with their + network domain in parenthesis, or the context is a public link even if the user is authenticated. + TODO in the latter case we should handle operations on behalf of the user with some scoped token, + but as for now we impersonate the owner we just consider public links as non-primary users. + ''' + return acctok['username'] != '' and '(' not in acctok['username'] and acctok['username'][-1] != ')' and \ + urlparse(acctok['appviewurl']).path.find(PUBLINKPREFIX) != 0 + + def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app, forcelock=False): '''Generates an access token for a given file and a given user, and returns a tuple with the file's inode and the URL-encoded access token.''' diff --git a/src/wopiserver.py b/src/wopiserver.py index cdb796ab..873875f5 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -173,8 +173,7 @@ def init(cls): except (configparser.NoOptionError, OSError, ValueError) as e: # any error we get here with the configuration is fatal cls.log.fatal('msg="Failed to initialize the service, aborting" error="%s"' % e) - print("Failed to initialize the service: %s\n" % e, file=sys.stderr) - sys.exit(22) + raise @classmethod def refreshconfig(cls): @@ -481,23 +480,23 @@ def wopiFilesPost(fileid): acctokOrMsg, httpcode = utils.validateAndLogHeaders(op) if httpcode: return acctokOrMsg, httpcode - if op not in ('GET_LOCK', 'PUT_USER_INFO') and utils.ViewMode(acctokOrMsg['viewmode']) != utils.ViewMode.READ_WRITE: - # protect this call if the WOPI client does not have privileges + if op == 'GET_LOCK': + return core.wopi.getLock(fileid, headers, acctokOrMsg) + if op == 'PUT_USER_INFO': + return core.wopi.putUserInfo(fileid, flask.request.get_data(), acctokOrMsg) + if op == 'PUT_RELATIVE': + return core.wopi.putRelative(fileid, headers, acctokOrMsg) + if utils.ViewMode(acctokOrMsg['viewmode']) != utils.ViewMode.READ_WRITE: + # the remaining operations require write privileges return 'Attempting to perform a write operation using a read-only token', http.client.UNAUTHORIZED if op in ('LOCK', 'REFRESH_LOCK'): return core.wopi.setLock(fileid, headers, acctokOrMsg) - if op == 'GET_LOCK': - return core.wopi.getLock(fileid, headers, acctokOrMsg) if op == 'UNLOCK': return core.wopi.unlock(fileid, headers, acctokOrMsg) - if op == 'PUT_RELATIVE': - return core.wopi.putRelative(fileid, headers, acctokOrMsg) if op == 'DELETE': return core.wopi.deleteFile(fileid, headers, acctokOrMsg) if op == 'RENAME_FILE': return core.wopi.renameFile(fileid, headers, acctokOrMsg) - if op == 'PUT_USER_INFO': - return core.wopi.putUserInfo(fileid, flask.request.get_data(), acctokOrMsg) # Any other op is unsupported Wopi.log.warning('msg="Unknown/unsupported operation" operation="%s"' % op) return 'Not supported operation found in header', http.client.NOT_IMPLEMENTED diff --git a/wopiserver.conf b/wopiserver.conf index 9b77b8c3..6d8d8d3b 100644 --- a/wopiserver.conf +++ b/wopiserver.conf @@ -110,10 +110,11 @@ wopilockexpiration = 1800 #detectexternallocks = True # Location of the user's personal space, used as a fall back location when storing -# webconflict files. By default, such files are stored in the same path -# as the original file. If that fails (e.g. because of missing permissions), +# PutRelative targets or webconflict files. Normally, such files are stored in the same +# path as the original file. If that fails (e.g. because of missing permissions), # an attempt is made to store such files in this path if specified, otherwise -# the system falls back to the recovery space (cf. io|recoverypath). +# the system falls back to the recovery space (cf. io|recoverypath) for web conflicts +# whereas PutRelative operations are just failed. # The keywords and are replaced with the actual username's # initial letter and the actual username, respectively, so you can use e.g. # /your_storage/home/user_initial/username From 2acdfa10df5202c08b22179bd7878340db131c8a Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 23 Feb 2023 23:10:49 +0100 Subject: [PATCH 174/325] Redefined wopiuser variable Now wopiuser is enforced to have the format `username!userid_as_returned_by_stat`, where the latter can be `username@idp` for CS3 interfaces or `uid:gid` for xroot. --- src/core/cs3iface.py | 6 +++--- src/core/localiface.py | 2 +- src/core/wopi.py | 10 +++++----- src/core/wopiutils.py | 16 ++++++++-------- src/core/xrootiface.py | 6 ++++-- src/wopiserver.py | 8 ++------ 6 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index 23d1005b..a44d962b 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -40,10 +40,10 @@ def init(inconfig, inlog): ctx['cs3gw'] = cs3gw_grpc.GatewayAPIStub(ch) -def getuseridfromcreds(token, _wopiuser): +def getuseridfromcreds(token, wopiuser): '''Maps a Reva token and wopiuser to the credentials to be used to access the storage. - For the CS3 API case, this is just the token''' - return token + For the CS3 API case this is the token, and wopiuser is expected to be `username!userid_as_returned_by_stat`''' + return token, wopiuser.split('@')[0] + '!' + wopiuser def _getcs3reference(endpoint, fileref): diff --git a/src/core/localiface.py b/src/core/localiface.py index 0eefacbd..10f848ef 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -70,7 +70,7 @@ def init(inconfig, inlog): def getuseridfromcreds(_token, _wopiuser): '''Maps a Reva token and wopiuser to the credentials to be used to access the storage. For the localfs case, this is trivially hardcoded''' - return '0:0' + return '0:0', 'root!0:0' def stat(_endpoint, filepath, _userid): diff --git a/src/core/wopi.py b/src/core/wopi.py index ff1c895e..f2ce239c 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -78,7 +78,7 @@ def checkFileInfo(fileid, acctok): fmd['BreadcrumbBrandName'] = srv.config.get('general', 'brandingname', fallback=None) fmd['BreadcrumbBrandUrl'] = srv.config.get('general', 'brandingurl', fallback=None) fmd['OwnerId'] = statInfo['ownerid'] - fmd['UserId'] = acctok['wopiuser'] # typically same as OwnerId; different when accessing shared documents + fmd['UserId'] = acctok['wopiuser'].split('!')[-1] # typically same as OwnerId; different when accessing shared documents fmd['Size'] = statInfo['size'] # note that in ownCloud 10 the version is generated as: `'V' + etag + checksum` fmd['Version'] = 'v%s' % statInfo['etag'] @@ -94,7 +94,7 @@ def checkFileInfo(fileid, acctok): fmd['SupportsContainers'] = False # TODO this is all to be implemented fmd['SupportsUserInfo'] = True uinfo = st.getxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], - utils.USERINFOKEY + '.' + acctok['wopiuser'].split('@')[0]) + utils.USERINFOKEY + '.' + acctok['wopiuser'].split('!')[0]) if uinfo: fmd['UserInfo'] = uinfo @@ -368,7 +368,7 @@ def putRelative(fileid, reqheaders, acctok): if acctok['viewmode'] != utils.ViewMode.READ_WRITE: # here we must have an authenticated user with no write rights on the current folder: go to the user's homepath targetName = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ - replace('username', acctok['wopiuser'].split('@')[0]) + os.path.sep + replace('username', acctok['wopiuser'].split('!')[0]) + os.path.sep else: targetName = os.path.dirname(acctok['filename']) if suggTarget: @@ -427,7 +427,7 @@ def putRelative(fileid, reqheaders, acctok): # make an attempt in the user's home if possible if utils.isPrimaryUser(acctok): targetName = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ - replace('username', acctok['wopiuser'].split('@')[0]) \ + replace('username', acctok['wopiuser'].split('!')[0]) \ + os.path.sep + os.path.basename(targetName) try: utils.storeWopiFile(acctok, None, utils.LASTSAVETIMEKEY, targetName) @@ -623,7 +623,7 @@ def putUserInfo(fileid, reqbody, acctok): lockmd = st.getlock(acctok['endpoint'], acctok['filename'], acctok['userid']) lockmd = (acctok['appname'], utils.encodeLock(lockmd)) if lockmd else None st.setxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], - utils.USERINFOKEY + '.' + acctok['wopiuser'].split('@')[0], reqbody.decode(), lockmd) + utils.USERINFOKEY + '.' + acctok['wopiuser'].split('!')[0], reqbody.decode(), lockmd) log.info('msg="PutUserInfo" user="%s" filename="%s" fileid="%s" token="%s"' % (acctok['userid'][-20:], acctok['filename'], fileid, flask.request.args['access_token'][-20:])) return 'OK', http.client.OK diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 4d86713e..91e365e8 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -211,7 +211,7 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app '''Generates an access token for a given file and a given user, and returns a tuple with the file's inode and the URL-encoded access token.''' appname, appediturl, appviewurl = app - username, wopiuser = user + friendlyname, wopiuser = user # wopiuser has the form `username!userid_in_stat_format` log.debug('msg="Generating token" userid="%s" fileid="%s" endpoint="%s" app="%s"' % (userid[-20:], fileid, endpoint, appname)) try: @@ -243,7 +243,7 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app viewmode = ViewMode.READ_ONLY tokmd = { 'userid': userid, 'wopiuser': wopiuser, 'filename': statinfo['filepath'], 'fileid': fileid, - 'username': username, 'viewmode': viewmode.value, 'folderurl': folderurl, 'endpoint': endpoint, + 'username': friendlyname, 'viewmode': viewmode.value, 'folderurl': folderurl, 'endpoint': endpoint, 'appname': appname, 'appediturl': appediturl, 'appviewurl': appviewurl, 'exp': exptime, 'iss': 'cs3org:wopiserver:%s' % WOPIVER # standard claims } @@ -252,9 +252,9 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app acctok = jwt.encode(tokmd, srv.wopisecret, algorithm='HS256') if 'MS 365' in appname: srv.allusers.add(userid) - log.info('msg="Access token generated" userid="%s" wopiuser="%s" username="%s" mode="%s" endpoint="%s" filename="%s" ' + log.info('msg="Access token generated" userid="%s" wopiuser="%s" friendlyname="%s" mode="%s" endpoint="%s" filename="%s" ' 'inode="%s" mtime="%s" folderurl="%s" appname="%s"%s expiration="%d" token="%s"' % - (userid[-20:], wopiuser, username, viewmode, endpoint, statinfo['filepath'], statinfo['inode'], statinfo['mtime'], + (userid[-20:], wopiuser, friendlyname, viewmode, endpoint, statinfo['filepath'], statinfo['inode'], statinfo['mtime'], folderurl, appname, ' forcelock="True"' if forcelock else '', exptime, acctok[-20:])) return statinfo['inode'], acctok, viewmode @@ -474,7 +474,7 @@ def makeLockSuccessResponse(operation, acctok, lock, oldlock, version): '''Generates and logs an HTTP 200 response with appropriate headers for Lock/RefreshLock operations''' session = flask.request.headers.get('X-WOPI-SessionId') if not session: - session = acctok['wopiuser'] + session = acctok['wopiuser'].split('!')[0] _resolveSession(session, acctok['filename']) log.info('msg="Successfully locked" lockop="%s" filename="%s" token="%s" sessionId="%s" ' @@ -494,7 +494,7 @@ def storeWopiFile(acctok, retrievedlock, xakey, targetname=''): targetname = acctok['filename'] session = flask.request.headers.get('X-WOPI-SessionId') if not session: - session = acctok['wopiuser'] + session = acctok['wopiuser'].split('!')[0] _resolveSession(session, targetname) writeerror = None @@ -529,7 +529,7 @@ def storeAfterConflict(acctok, retrievedlock, lock, reason): else: # let's try the configured user's (or owner's) homepath instead of the current folder newname = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ - replace('username', acctok['wopiuser'].split('@')[0]) \ + replace('username', acctok['wopiuser'].split('!')[0]) \ + os.path.sep + os.path.basename(newname) try: storeWopiFile(acctok, retrievedlock, LASTSAVETIMEKEY, newname) @@ -554,7 +554,7 @@ def storeAfterConflict(acctok, retrievedlock, lock, reason): def storeForRecovery(content, wopiuser, filename, acctokforlog, exception): try: filepath = srv.recoverypath + os.sep + time.strftime('%Y%m%dT%H%M%S') + '_editedby_' \ - + secure_filename(wopiuser.split('@')[0]) + '_origat_' + secure_filename(filename) + + secure_filename(wopiuser.split('!')[0]) + '_origat_' + secure_filename(filename) with open(filepath, mode='wb') as f: written = f.write(content) if written != len(content): diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 477c6a30..c7976df8 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -156,9 +156,11 @@ def _getfilepath(filepath, encodeamp=False): def getuseridfromcreds(_token, wopiuser): '''Maps a Reva token and wopiuser to the credentials to be used to access the storage. - For the xrootd case, we have to resolve the username to uid:gid''' + For the xrootd case, we have to resolve the username to uid:gid and return + username!uid:gid as wopiuser in order to respect the format `username!userid_as_returned_by_stat`''' userid = getpwnam(wopiuser.split('@')[0]) # a wopiuser has the form username@idp - return str(userid.pw_uid) + ':' + str(userid.pw_gid) + userid = str(userid.pw_uid) + ':' + str(userid.pw_gid) + return userid, wopiuser.split('@')[0] + '!' + userid def stat(endpoint, filepath, userid): diff --git a/src/wopiserver.py b/src/wopiserver.py index 873875f5..efbcd0fe 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -329,11 +329,7 @@ def iopOpenInApp(): return 'Missing appname or appurl arguments', http.client.BAD_REQUEST try: - userid = storage.getuseridfromcreds(usertoken, wopiuser) - if userid != usertoken: - # this happens in hybrid deployments with xrootd as storage interface: - # in this case we decorate the wopiuser with the resolved uid:gid - wopiuser += ':' + userid + userid, wopiuser = storage.getuseridfromcreds(usertoken, wopiuser) inode, acctok, vm = utils.generateAccessToken(userid, fileid, viewmode, (username, wopiuser), folderurl, endpoint, (appname, appurl, appviewurl), forcelock=forcelock) except IOError as e: @@ -438,7 +434,7 @@ def iopWopiTest(): return 'Missing arguments', http.client.BAD_REQUEST if Wopi.useHttps: return 'WOPI validator not supported in https mode', http.client.BAD_REQUEST - inode, acctok, _ = utils.generateAccessToken(usertoken, filepath, utils.ViewMode.READ_WRITE, ('test', usertoken), + inode, acctok, _ = utils.generateAccessToken(usertoken, filepath, utils.ViewMode.READ_WRITE, ('test', 'test!' + usertoken), 'http://folderurlfortestonly/', endpoint, ('WOPI validator', 'http://fortestonly/', 'http://fortestonly/')) Wopi.log.info('msg="iopWopiTest: preparing test via WOPI validator" client="%s"' % req.remote_addr) From b866c68802c556439ade2ce7ff828d976478e94e Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 24 Feb 2023 10:21:00 +0100 Subject: [PATCH 175/325] Minor changes --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3c4fe9a1..6fd5c1fa 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ Contributors (oldest contributions first): - Michael Barz (@micbar) - Robert Kaussow (@xoxys) - Javier Ferrer (@javfg) - Vasco Guita (@vascoguita) - Tomas Zaluckij (@Tomaszal) +- Vasco Guita (@vascoguita) +- Tomas Zaluckij (@Tomaszal) Initial revision: December 2016
First production version for CERNBox: September 2017 (presented at [oCCon17](https://occon17.owncloud.org) - [slides](https://www.slideshare.net/giuseppelopresti/collaborative-editing-and-more-in-cernbox))
@@ -72,7 +72,8 @@ To run the tests, either run `pytest` if available in your system, or execute th ### Test using the Microsoft WOPI validator test suite -This is work in progress. Refer to [these notes](test/wopi-validator.md). +Refer to [these notes](test/wopi-validator.md). Microsoft also provides a graphical version of the test suite +as part of their Office 365 offer, which is also supported via the Reva open-in-app workflow. ## Run the WOPI server locally for development purposes @@ -89,8 +90,10 @@ This is work in progress. Refer to [these notes](test/wopi-validator.md). ### Test the open-in-app workflow on the local WOPI server Once the WOPI server runs on top of local storage, the `tools/wopiopen.py` script can be used -to test the open-in-app workflow. For that, assuming you have e.g. CodiMD deployed in your (docker-compose) cluster: +to test the open-in-app workflow. +For that, assuming you have e.g. CodiMD deployed in your (docker-compose) cluster: 1. Create a `test.md` file in your local storage folder, e.g. `/var/wopi_local_storage` 2. From the WOPI server folder, execute `tools/wopiopen.py -a CodiMD -i "internal_CodiMD_URL" -u "user_visible_CodiMD_URL" -k CodiMD_API_Key test.md` 3. If everything was setup correctly, you'll get a JSON response including an `app-url`. Open it in a browser to access the file. Otherwise, the tool prints the response from the WOPI server and the logs should help troubleshooting the problem. + From 3d67db8acab8bb69d82a7580f3636bb74c081e78 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 24 Feb 2023 11:06:37 +0100 Subject: [PATCH 176/325] Fixed install for apt-based images shell-based if statements do not seem to work within RUN, this refactoring works without them --- wopiserver.Dockerfile | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/wopiserver.Dockerfile b/wopiserver.Dockerfile index b0280e91..359f36a2 100644 --- a/wopiserver.Dockerfile +++ b/wopiserver.Dockerfile @@ -14,13 +14,8 @@ LABEL maintainer="cernbox-admins@cern.ch" \ # prerequisites: we explicitly install g++ as it is required by grpcio but missing from its dependencies WORKDIR /app COPY requirements.txt . -RUN if command -v apk &> /dev/null; then \ - echo "Using apk"; apk add curl g++; \ - elif command -v apt &> /dev/null; then \ - echo "Using apt"; apt -y install g++; \ - else \ - echo "This distribution does not provide a supported package manager"; false; \ - fi +RUN command -v apk && apk add curl g++ || true +RUN command -v apt && apt update && apt -y install curl g++ || true RUN pip3 install --upgrade pip setuptools && \ pip3 install --no-cache-dir --upgrade -r requirements.txt From ceb14ded619e274c9e93faa99eaea96a6ea422a8 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 24 Feb 2023 15:14:57 +0100 Subject: [PATCH 177/325] Fixed PutRelative and improved logs --- src/core/wopi.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index f2ce239c..1833675a 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -78,7 +78,7 @@ def checkFileInfo(fileid, acctok): fmd['BreadcrumbBrandName'] = srv.config.get('general', 'brandingname', fallback=None) fmd['BreadcrumbBrandUrl'] = srv.config.get('general', 'brandingurl', fallback=None) fmd['OwnerId'] = statInfo['ownerid'] - fmd['UserId'] = acctok['wopiuser'].split('!')[-1] # typically same as OwnerId; different when accessing shared documents + fmd['UserId'] = acctok['wopiuser'].split('!')[-1] # typically same as OwnerId; different when accessing shared documents fmd['Size'] = statInfo['size'] # note that in ownCloud 10 the version is generated as: `'V' + etag + checksum` fmd['Version'] = 'v%s' % statInfo['etag'] @@ -360,15 +360,18 @@ def putRelative(fileid, reqheaders, acctok): overwriteTarget = str(reqheaders.get('X-WOPI-OverwriteRelativeTarget')).upper() == 'TRUE' log.info('msg="PutRelative" user="%s" filename="%s" fileid="%s" suggTarget="%s" relTarget="%s" ' 'overwrite="%r" wopitimestamp="%s" token="%s"' % - (acctok['userid'], acctok['filename'], fileid, suggTarget, relTarget, + (acctok['userid'][-20:], acctok['filename'], fileid, suggTarget, relTarget, overwriteTarget, reqheaders.get('X-WOPI-TimeStamp'), flask.request.args['access_token'][-20:])) # either one xor the other MUST be present; note we can't use `^` as we have a mix of str and NoneType if (suggTarget and relTarget) or (not suggTarget and not relTarget): return 'Conflicting headers given', http.client.BAD_REQUEST - if acctok['viewmode'] != utils.ViewMode.READ_WRITE: + if utils.ViewMode(acctok['viewmode']) != utils.ViewMode.READ_WRITE: # here we must have an authenticated user with no write rights on the current folder: go to the user's homepath targetName = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ replace('username', acctok['wopiuser'].split('!')[0]) + os.path.sep + log.info('msg="PutRelative: set homepath as destination" user="%s" viewmode="%s" filename="%s" target="%s" token="%s"' % + (acctok['userid'][-20:], acctok['viewmode'], acctok['filename'], targetName, + flask.request.args['access_token'][-20:])) else: targetName = os.path.dirname(acctok['filename']) if suggTarget: @@ -390,7 +393,7 @@ def putRelative(fileid, reqheaders, acctok): # OK, the targetName is good to go break # we got another error with this file, fail - log.error('msg="PutRelative" user="%s" filename="%s" token="%s" suggTarget="%s" error="%s"' % + log.error('msg="Error in PutRelative" user="%s" filename="%s" token="%s" suggTarget="%s" error="%s"' % (acctok['userid'][-20:], targetName, flask.request.args['access_token'][-20:], suggTarget, str(e))) return 'Error with the given target', http.client.INTERNAL_SERVER_ERROR @@ -429,6 +432,8 @@ def putRelative(fileid, reqheaders, acctok): targetName = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ replace('username', acctok['wopiuser'].split('!')[0]) \ + os.path.sep + os.path.basename(targetName) + log.info('msg="PutRelative: set homepath as destination" user="%s" filename="%s" target="%s" token="%s"' % + (acctok['userid'][-20:], acctok['filename'], targetName, flask.request.args['access_token'][-20:])) try: utils.storeWopiFile(acctok, None, utils.LASTSAVETIMEKEY, targetName) raisenoaccess = False From b50dc76ff3678df06b9137efb52f1e402b57075d Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 24 Feb 2023 17:57:26 +0100 Subject: [PATCH 178/325] Fixed PutUserInfo for CS3 storages --- src/core/wopi.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 1833675a..7e959268 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -93,7 +93,7 @@ def checkFileInfo(fileid, acctok): fmd['SupportsRename'] = fmd['UserCanRename'] = enablerename and (acctok['viewmode'] == utils.ViewMode.READ_WRITE) fmd['SupportsContainers'] = False # TODO this is all to be implemented fmd['SupportsUserInfo'] = True - uinfo = st.getxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], + uinfo = st.getxattr(acctok['endpoint'], acctok['filename'], acctok['userid'], utils.USERINFOKEY + '.' + acctok['wopiuser'].split('!')[0]) if uinfo: fmd['UserInfo'] = uinfo @@ -624,10 +624,9 @@ def putFile(fileid, acctok): def putUserInfo(fileid, reqbody, acctok): '''Implements the PutUserInfo WOPI call''' try: - statInfo = st.statx(acctok['endpoint'], acctok['filename'], acctok['userid']) lockmd = st.getlock(acctok['endpoint'], acctok['filename'], acctok['userid']) lockmd = (acctok['appname'], utils.encodeLock(lockmd)) if lockmd else None - st.setxattr(acctok['endpoint'], acctok['filename'], statInfo['ownerid'], + st.setxattr(acctok['endpoint'], acctok['filename'], acctok['userid'], utils.USERINFOKEY + '.' + acctok['wopiuser'].split('!')[0], reqbody.decode(), lockmd) log.info('msg="PutUserInfo" user="%s" filename="%s" fileid="%s" token="%s"' % (acctok['userid'][-20:], acctok['filename'], fileid, flask.request.args['access_token'][-20:])) From 85fa7bb14d19525aee4b248b14cd64f14ef9a1d3 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 21 Feb 2023 18:14:11 +0100 Subject: [PATCH 179/325] cs3iface: allow using absolute paths when the storage driver returns them --- src/core/cs3iface.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index a44d962b..d23cd3ce 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -103,9 +103,13 @@ def stat(endpoint, fileref, userid, versioninv=1): raise IOError('Unexpected type %d' % statInfo.info.type) inode = common.encodeinode(statInfo.info.id.storage_id, statInfo.info.id.opaque_id) - # here we build an hybrid path that can be used to reference the file, as the path is actually just the basename - # (and eventually the CS3 APIs should be updated to reflect that): note that as per specs the parent_id MUST be available - filepath = statInfo.info.parent_id.opaque_id + '/' + os.path.basename(statInfo.info.path) + if statInfo.info.path[0] == '/': + # we got an absolute path from Reva, use it + filepath = statInfo.info.path + else: + # we got a relative path (actually, just the basename): build an hybrid path that can be used to reference + # the file, using the parent_id that per specs MUST be available + filepath = statInfo.info.parent_id.opaque_id + '/' + os.path.basename(statInfo.info.path) log.info('msg="Invoked stat" fileref="%s" trace="%s" inode="%s" filepath="%s" elapsedTimems="%.1f"' % (fileref, statInfo.status.trace, inode, filepath, (tend-tstart)*1000)) return { From 172c89de4d347ae7ef8182e856e56723e16392b0 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 24 Feb 2023 17:38:09 +0100 Subject: [PATCH 180/325] Improved logging --- src/core/cs3iface.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index d23cd3ce..ce573ed6 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -89,15 +89,17 @@ def stat(endpoint, fileref, userid, versioninv=1): ref = _getcs3reference(endpoint, fileref) statInfo = ctx['cs3gw'].Stat(request=cs3sp.StatRequest(ref=ref), metadata=[('x-access-token', userid)]) tend = time.time() - if statInfo.status.code != cs3code.CODE_OK: - log.info('msg="Failed stat" fileref="%s" trace="%s" reason="%s"' % - (fileref, statInfo.status.trace, statInfo.status.message.replace('"', "'"))) - raise IOError(common.ENOENT_MSG if statInfo.status.code == cs3code.CODE_NOT_FOUND else statInfo.status.message) + if statInfo.status.code == cs3code.CODE_NOT_FOUND: + log.info('msg="File not found" fileref="%s" trace="%s"' % (fileref, statInfo.status.trace)) + raise IOError(common.ENOENT_MSG) + if statInfo.status.code != cs3code.CODE_OK: + log.error('msg="Failed stat" fileref="%s" trace="%s" reason="%s"' % + (fileref, statInfo.status.trace, statInfo.status.message.replace('"', "'"))) + raise IOError(statInfo.status.message) if statInfo.info.type == cs3spr.RESOURCE_TYPE_CONTAINER: log.info('msg="Invoked stat" fileref="%s" trace="%s" result="ISDIR"' % (fileref, statInfo.status.trace)) raise IOError('Is a directory') - if statInfo.info.type not in (cs3spr.RESOURCE_TYPE_FILE, cs3spr.RESOURCE_TYPE_SYMLINK): log.warning('msg="Invoked stat" fileref="%s" unexpectedtype="%d"' % (fileref, statInfo.info.type)) raise IOError('Unexpected type %d' % statInfo.info.type) @@ -154,8 +156,8 @@ def getxattr(endpoint, filepath, userid, key): log.debug('msg="Invoked stat for getxattr on missing file" filepath="%s"' % filepath) return None if statInfo.status.code != cs3code.CODE_OK: - log.error('msg="Failed to stat" filepath="%s" trace="%s" key="%s" reason="%s"' % - (filepath, statInfo.status.trace, key, statInfo.status.message.replace('"', "'"))) + log.error('msg="Failed to stat" filepath="%s" userid="%s" trace="%s" key="%s" reason="%s"' % + (filepath, userid[-20:], statInfo.status.trace, key, statInfo.status.message.replace('"', "'"))) raise IOError(statInfo.status.message) try: xattrvalue = statInfo.info.arbitrary_metadata.metadata[key] @@ -164,8 +166,8 @@ def getxattr(endpoint, filepath, userid, key): log.debug('msg="Invoked stat for getxattr" filepath="%s" elapsedTimems="%.1f"' % (filepath, (tend - tstart) * 1000)) return xattrvalue except KeyError: - log.warning('msg="Empty value or key not found in getxattr" filepath="%s" key="%s" trace="%s" metadata="%s"' % - (filepath, key, statInfo.status.trace, statInfo.info.arbitrary_metadata.metadata)) + log.info('msg="Empty value or key not found in getxattr" filepath="%s" key="%s" trace="%s" metadata="%s"' % + (filepath, key, statInfo.status.trace, statInfo.info.arbitrary_metadata.metadata)) return None From 0a93b8b298fc39e900f170a9a0e1cdc500b9b9b7 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 28 Feb 2023 16:45:19 +0100 Subject: [PATCH 181/325] Linting --- src/core/wopi.py | 5 ++--- src/core/wopiutils.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 7e959268..c529904a 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -88,8 +88,7 @@ def checkFileInfo(fileid, acctok): # SaveAs functionality is disabled for anonymous and federated users when in read-only mode, as they have # no personal space where to save as an alternate location. # Note that single-file r/w shares are optimistically offered a SaveAs option, which may only work for primary users. - fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE \ - and not utils.isPrimaryUser(acctok) + fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE and not utils.isPrimaryUser(acctok) fmd['SupportsRename'] = fmd['UserCanRename'] = enablerename and (acctok['viewmode'] == utils.ViewMode.READ_WRITE) fmd['SupportsContainers'] = False # TODO this is all to be implemented fmd['SupportsUserInfo'] = True @@ -431,7 +430,7 @@ def putRelative(fileid, reqheaders, acctok): if utils.isPrimaryUser(acctok): targetName = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ replace('username', acctok['wopiuser'].split('!')[0]) \ - + os.path.sep + os.path.basename(targetName) + + os.path.sep + os.path.basename(targetName) # noqa: E131 log.info('msg="PutRelative: set homepath as destination" user="%s" filename="%s" target="%s" token="%s"' % (acctok['userid'][-20:], acctok['filename'], targetName, flask.request.args['access_token'][-20:])) try: diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 91e365e8..9dc7f751 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -254,8 +254,8 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app srv.allusers.add(userid) log.info('msg="Access token generated" userid="%s" wopiuser="%s" friendlyname="%s" mode="%s" endpoint="%s" filename="%s" ' 'inode="%s" mtime="%s" folderurl="%s" appname="%s"%s expiration="%d" token="%s"' % - (userid[-20:], wopiuser, friendlyname, viewmode, endpoint, statinfo['filepath'], statinfo['inode'], statinfo['mtime'], - folderurl, appname, ' forcelock="True"' if forcelock else '', exptime, acctok[-20:])) + (userid[-20:], wopiuser, friendlyname, viewmode, endpoint, statinfo['filepath'], statinfo['inode'], + statinfo['mtime'], folderurl, appname, ' forcelock="True"' if forcelock else '', exptime, acctok[-20:])) return statinfo['inode'], acctok, viewmode @@ -530,7 +530,7 @@ def storeAfterConflict(acctok, retrievedlock, lock, reason): # let's try the configured user's (or owner's) homepath instead of the current folder newname = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ replace('username', acctok['wopiuser'].split('!')[0]) \ - + os.path.sep + os.path.basename(newname) + + os.path.sep + os.path.basename(newname) # noqa: E131 try: storeWopiFile(acctok, retrievedlock, LASTSAVETIMEKEY, newname) except IOError as e: From 9cc14e1c2613aa0f67a64343e0b11f69772b65e9 Mon Sep 17 00:00:00 2001 From: Vasco Guita Date: Wed, 1 Mar 2023 14:58:10 +0100 Subject: [PATCH 182/325] Fix release workflow (#114) --- .github/workflows/release.yml | 68 ++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2ab80424..adfc6563 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,12 +9,30 @@ jobs: # The following is a clone of cs3org/reva/.github/workflows/docker.yml because reusable actions do not (yet) support lists as input types: # see https://github.com/community/community/discussions/11692 release: - runs-on: self-hosted + runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'cs3org/wopiserver'] }} + strategy: + fail-fast: false + matrix: + include: + - file: wopiserver.Dockerfile + tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-amd64 + platform: linux/amd64 + image: python:3.11-alpine + push: ${{ github.event_name != 'workflow_dispatch' }} + - file: wopiserver.Dockerfile + tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-arm64 + platform: linux/arm64 + image: python:3.10-slim-buster + push: ${{ github.event_name != 'workflow_dispatch' }} + - file: wopiserver-xrootd.Dockerfile + tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-xrootd + platform: linux/amd64 + push: ${{ github.event_name != 'workflow_dispatch' }} steps: - name: Checkout uses: actions/checkout@v3.1.0 - name: Set up QEMU - if: matrix.platforms != '' + if: matrix.platform != '' uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -34,30 +52,22 @@ jobs: build-args: | VERSION=${{ github.ref_name }} BASEIMAGE=${{ matrix.image }} - platforms: ${{ matrix.platforms }} -# - name: Upload ${{ matrix.tags }} Docker image to artifacts -# uses: ishworkh/docker-image-artifact-upload@v1 -# if: ${{ inputs.load }} -# with: -# image: ${{ inputs.tags }} -# retention_days: '1' -# end of the clone - - strategy: - fail-fast: false - matrix: - include: - - file: wopiserver.Dockerfile - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }},${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:latest - platform: linux/amd64 - image: python:3.11-alpine - push: ${{ github.event_name != 'workflow_dispatch' }} - - file: wopiserver.Dockerfile - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }},${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:latest - platform: linux/arm64 - image: python:3.10-slim-buster - push: ${{ github.event_name != 'workflow_dispatch' }} - - file: wopiserver-xrootd.Dockerfile - tags: ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-xrootd - platform: linux/amd64 - push: ${{ github.event_name != 'workflow_dispatch' }} + platforms: ${{ matrix.platform }} + build-multi-architecture: + runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'cs3org/wopiserver'] }} + needs: release + if: github.event_name != 'workflow_dispatch' + steps: + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - uses: int128/docker-manifest-create-action@v1 + with: + tags: | + ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }} + ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:latest + suffixes: | + -amd64 + -arm64 \ No newline at end of file From f90f9fcc20ec6834482e2f7038df32174433d43e Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 1 Mar 2023 15:34:52 +0100 Subject: [PATCH 183/325] Linting --- src/wopiserver.py | 1 - tools/wopiopen.py | 10 ++++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/wopiserver.py b/src/wopiserver.py index efbcd0fe..88400931 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -560,7 +560,6 @@ def cboxOpen_deprecated(): return UNAUTHORIZED # now validate the user identity and deny root access try: - userid = 'N/A' ruid = int(req.args['ruid']) rgid = int(req.args['rgid']) userid = '%d:%d' % (ruid, rgid) diff --git a/tools/wopiopen.py b/tools/wopiopen.py index 46038b5e..44c8eeeb 100755 --- a/tools/wopiopen.py +++ b/tools/wopiopen.py @@ -13,20 +13,22 @@ import configparser import requests sys.path.append('src') # for tests out of the git repo -from core.wopiutils import ViewMode +from core.wopiutils import ViewMode # noqa: E402 # usage function def usage(exitcode): '''Prints usage''' - print('Usage : ' + sys.argv[0] + ' -a|--appname -u|--appurl [-i|--appinturl ] -k|--apikey ' - '[-s|--storage ] [-v|--viewmode VIEW_ONLY|READ_ONLY|READ_WRITE|PREVIEW] [-x|--x-access-token ] ') + print('Usage : ' + sys.argv[0] + ' -a|--appname -u|--appurl [-i|--appinturl ] ' + '-k|--apikey [-s|--storage ] [-v|--viewmode VIEW_ONLY|READ_ONLY|READ_WRITE|PREVIEW] ' + '[-x|--x-access-token ] ') sys.exit(exitcode) # first parse the options try: - options, args = getopt.getopt(sys.argv[1:], 'hv:s:a:i:u:x:k:', ['help', 'viewmode', 'storage', 'appname', 'appinturl', 'appurl', 'x-access-token', 'apikey']) + options, args = getopt.getopt(sys.argv[1:], 'hv:s:a:i:u:x:k:', + ['help', 'viewmode', 'storage', 'appname', 'appinturl', 'appurl', 'x-access-token', 'apikey']) except getopt.GetoptError as e: print(e) usage(1) From 91cf3997ee599baa8ec3c004b6e9d1a4ba991b88 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 1 Mar 2023 15:39:28 +0100 Subject: [PATCH 184/325] cs3iface: more logging in Stat --- src/core/cs3iface.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index ce573ed6..2fef99e8 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -91,17 +91,19 @@ def stat(endpoint, fileref, userid, versioninv=1): tend = time.time() if statInfo.status.code == cs3code.CODE_NOT_FOUND: - log.info('msg="File not found" fileref="%s" trace="%s"' % (fileref, statInfo.status.trace)) + log.info('msg="File not found" endpoint="%s" fileref="%s" trace="%s"' % (endpoint, fileref, statInfo.status.trace)) raise IOError(common.ENOENT_MSG) if statInfo.status.code != cs3code.CODE_OK: - log.error('msg="Failed stat" fileref="%s" trace="%s" reason="%s"' % - (fileref, statInfo.status.trace, statInfo.status.message.replace('"', "'"))) + log.error('msg="Failed stat" endpoint="%s" fileref="%s" trace="%s" reason="%s"' % + (endpoint, fileref, statInfo.status.trace, statInfo.status.message.replace('"', "'"))) raise IOError(statInfo.status.message) if statInfo.info.type == cs3spr.RESOURCE_TYPE_CONTAINER: - log.info('msg="Invoked stat" fileref="%s" trace="%s" result="ISDIR"' % (fileref, statInfo.status.trace)) + log.info('msg="Invoked stat" endpoint="%s" fileref="%s" trace="%s" result="ISDIR"' % + (endpoint, fileref, statInfo.status.trace)) raise IOError('Is a directory') if statInfo.info.type not in (cs3spr.RESOURCE_TYPE_FILE, cs3spr.RESOURCE_TYPE_SYMLINK): - log.warning('msg="Invoked stat" fileref="%s" unexpectedtype="%d"' % (fileref, statInfo.info.type)) + log.warning('msg="Invoked stat" endpoint="%s" fileref="%s" unexpectedtype="%d"' % + (endpoint, fileref, statInfo.info.type)) raise IOError('Unexpected type %d' % statInfo.info.type) inode = common.encodeinode(statInfo.info.id.storage_id, statInfo.info.id.opaque_id) From 311a5dbdaf3faedb087122053a9dde2d07712f57 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sun, 5 Mar 2023 19:22:09 +0100 Subject: [PATCH 185/325] Added support for app user type This allows a better control flow for external/federated users, cf. also https://github.com/cs3org/reva/pull/36941 --- src/core/wopi.py | 19 +++++++++---------- src/core/wopiutils.py | 35 +++++++++++++++++------------------ src/wopiserver.py | 12 ++++++++++-- 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index c529904a..fef1461f 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -58,23 +58,21 @@ def checkFileInfo(fileid, acctok): furl = acctok['folderurl'] if furl != '/': fmd['BreadcrumbFolderUrl'] = furl + '?scrollTo=' + fmd['BaseFileName'] - if acctok['username'] == '': + if acctok['username'] == '' or acctok['usertype'] == utils.UserType.ANONYMOUS: fmd['IsAnonymousUser'] = True fmd['UserFriendlyName'] = 'Guest ' + utils.randomString(3) - if furl != '/': - fmd['BreadcrumbFolderName'] = 'Public share' + fmd['BreadcrumbFolderName'] = 'Public share' else: fmd['IsAnonymousUser'] = False fmd['UserFriendlyName'] = acctok['username'] - if furl != '/': - fmd['BreadcrumbFolderName'] = 'Parent folder' + fmd['BreadcrumbFolderName'] = 'ScienceMesh share' if acctok['usertype'] == utils.UserType.OCM else 'Parent folder' if acctok['viewmode'] in (utils.ViewMode.READ_ONLY, utils.ViewMode.READ_WRITE) \ and srv.config.get('general', 'downloadurl', fallback=None): fmd['DownloadUrl'] = fmd['FileUrl'] = '%s?access_token=%s' % \ (srv.config.get('general', 'downloadurl'), flask.request.args['access_token']) if srv.config.get('general', 'businessflow', fallback='False').upper() == 'TRUE': - # enable the check for real users, not for public links - fmd['LicenseCheckForEditIsEnabled'] = not fmd['IsAnonymousUser'] + # enable the check for real users, not for public links / federated access + fmd['LicenseCheckForEditIsEnabled'] = acctok['usertype'] == utils.UserType.REGULAR fmd['BreadcrumbBrandName'] = srv.config.get('general', 'brandingname', fallback=None) fmd['BreadcrumbBrandUrl'] = srv.config.get('general', 'brandingurl', fallback=None) fmd['OwnerId'] = statInfo['ownerid'] @@ -87,8 +85,9 @@ def checkFileInfo(fileid, acctok): fmd['SupportsDeleteFile'] = acctok['viewmode'] == utils.ViewMode.READ_WRITE # SaveAs functionality is disabled for anonymous and federated users when in read-only mode, as they have # no personal space where to save as an alternate location. - # Note that single-file r/w shares are optimistically offered a SaveAs option, which may only work for primary users. - fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE and not utils.isPrimaryUser(acctok) + # Note that single-file r/w shares are optimistically offered a SaveAs option, which may only work for regular users. + fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE and \ + acctok['usertype'] != utils.UserType.REGULAR fmd['SupportsRename'] = fmd['UserCanRename'] = enablerename and (acctok['viewmode'] == utils.ViewMode.READ_WRITE) fmd['SupportsContainers'] = False # TODO this is all to be implemented fmd['SupportsUserInfo'] = True @@ -427,7 +426,7 @@ def putRelative(fileid, reqheaders, acctok): return IO_ERROR, http.client.INTERNAL_SERVER_ERROR raisenoaccess = True # make an attempt in the user's home if possible - if utils.isPrimaryUser(acctok): + if acctok['usertype'] == utils.UserType.REGULAR: targetName = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ replace('username', acctok['wopiuser'].split('!')[0]) \ + os.path.sep + os.path.basename(targetName) # noqa: E131 diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 9dc7f751..42ff5b89 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -17,7 +17,7 @@ from datetime import datetime from base64 import b64encode, b64decode from binascii import Error as B64Error -from urllib.parse import quote_plus as url_quote_plus, urlparse +from urllib.parse import quote_plus as url_quote_plus import http.client import flask import jwt @@ -33,9 +33,6 @@ # header used by reverse proxies such as traefik to pass the real remote IP address REALIPHEADER = 'X-Real-IP' -# the prefix used for public links, cf. reva:pkg/app/provider/wopi/wopi.go -PUBLINKPREFIX = '/files/link/public' - # convenience references to global entities st = None srv = None @@ -57,6 +54,20 @@ class ViewMode(Enum): # The file can be downloaded and updated, and the app should be shown in preview mode PREVIEW = "VIEW_MODE_PREVIEW" +class UserType(Enum): + '''App user types as given by + https://github.com/cs3org/reva/blob/master/pkg/app/provider/wopi/wopi.go + ''' + INVALID = "invalid" + # regular user, logged in the local ID provider + REGULAR = "regular" + # federated/external user, logged in the local ID provider but with no home space + FEDERATED = "federated" + # OCM user, logged in a remote ID provider + OCM = "ocm" + # anonymous user, accessing a public link + ANONYMOUS = "anonymous" + class JsonLogger: '''A wrapper class in front of a logger, based on the facade pattern''' @@ -195,23 +206,11 @@ def randomString(size): return ''.join([choice(ascii_lowercase) for _ in range(size)]) -def isPrimaryUser(acctok): - '''Return whether a session belongs to a primary/internal account, in the context of - saving a webconflicted file or a SaveAs operation. - False means the user is either anonymous or federated/external, which is given by Reva with their - network domain in parenthesis, or the context is a public link even if the user is authenticated. - TODO in the latter case we should handle operations on behalf of the user with some scoped token, - but as for now we impersonate the owner we just consider public links as non-primary users. - ''' - return acctok['username'] != '' and '(' not in acctok['username'] and acctok['username'][-1] != ')' and \ - urlparse(acctok['appviewurl']).path.find(PUBLINKPREFIX) != 0 - - def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app, forcelock=False): '''Generates an access token for a given file and a given user, and returns a tuple with the file's inode and the URL-encoded access token.''' appname, appediturl, appviewurl = app - friendlyname, wopiuser = user # wopiuser has the form `username!userid_in_stat_format` + friendlyname, wopiuser, usertype = user # wopiuser has the form `username!userid_in_stat_format` log.debug('msg="Generating token" userid="%s" fileid="%s" endpoint="%s" app="%s"' % (userid[-20:], fileid, endpoint, appname)) try: @@ -242,7 +241,7 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app log.info('msg="Forcing read-only access to ODF file" filename="%s"' % statinfo['filepath']) viewmode = ViewMode.READ_ONLY tokmd = { - 'userid': userid, 'wopiuser': wopiuser, 'filename': statinfo['filepath'], 'fileid': fileid, + 'userid': userid, 'wopiuser': wopiuser, 'usertype': usertype.value, 'filename': statinfo['filepath'], 'fileid': fileid, 'username': friendlyname, 'viewmode': viewmode.value, 'folderurl': folderurl, 'endpoint': endpoint, 'appname': appname, 'appediturl': appediturl, 'appviewurl': appviewurl, 'exp': exptime, 'iss': 'cs3org:wopiserver:%s' % WOPIVER # standard claims diff --git a/src/wopiserver.py b/src/wopiserver.py index 88400931..001d861a 100755 --- a/src/wopiserver.py +++ b/src/wopiserver.py @@ -283,6 +283,8 @@ def iopOpenInApp(): - string appviewurl (optional): the URL of the end-user application in view mode when different (defaults to appurl) - string appinturl (optional): the internal URL of the end-user application (applicable with containerized deployments) - string forcelock (optional): if present, will force the lock when possible to work around MS Office issues + - string usertype (optional): one of "regular", "federated", "ocm", "anonymous". Defaults to "regular" + Returns: a JSON response as follows: { "app-url" : "", @@ -324,14 +326,20 @@ def iopOpenInApp(): appurl = url_unquote_plus(req.args.get('appurl', '')).strip('/') appviewurl = url_unquote_plus(req.args.get('appviewurl', appurl)).strip('/') forcelock = req.args.get('forcelock', False) + try: + usertype = utils.UserType(req.args.get('usertype', utils.UserType.REGULAR)) + except (KeyError, ValueError) as e: + Wopi.log.warning('msg="iopOpenInApp: invalid usertype, falling back to regular" client="%s" usertype="%s" error="%s"' % + (req.remote_addr, req.args.get('usertype'), e)) + usertype = utils.UserType.REGULAR if not appname or not appurl: Wopi.log.warning('msg="iopOpenInApp: app-related arguments must be provided" client="%s"' % req.remote_addr) return 'Missing appname or appurl arguments', http.client.BAD_REQUEST try: userid, wopiuser = storage.getuseridfromcreds(usertoken, wopiuser) - inode, acctok, vm = utils.generateAccessToken(userid, fileid, viewmode, (username, wopiuser), folderurl, endpoint, - (appname, appurl, appviewurl), forcelock=forcelock) + inode, acctok, vm = utils.generateAccessToken(userid, fileid, viewmode, (username, wopiuser, usertype), folderurl, + endpoint, (appname, appurl, appviewurl), forcelock=forcelock) except IOError as e: Wopi.log.info('msg="iopOpenInApp: remote error on generating token" client="%s" user="%s" ' 'friendlyname="%s" mode="%s" endpoint="%s" reason="%s"' % From d16bad697996f0a3939dfaffeeeee488e53a6658 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 10 Mar 2023 10:18:29 +0100 Subject: [PATCH 186/325] New release --- CHANGELOG.md | 11 +++++++++++ README.md | 1 - src/core/wopiutils.py | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 647afae6..68bde8fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ ## Changelog for the WOPI server +### Fri 10 Mar 2023 - v9.5.0 +- Introduced concept of user type, given on `/wopi/iop/open`, + to better serve federated vs regular users with respect to + folder URLs and SaveAs operations +- Redefined `conflictpath` option as `homepath` (the former is + still supported for backwards compatibility): when defined, + a SaveAs operation falls back to the user's home when it + can't work on the original folder +- Fixed PutUserInfo to use the user's username as xattr key +- Added arm64-based builds + ### Tue Jan 31 2023 - v9.4.0 - Introduced support to forcefully evict valid locks to compensate Microsoft Online mishandling of collaborative diff --git a/README.md b/README.md index 6fd5c1fa..174c6892 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,6 @@ Contributors (oldest contributions first): - Robert Kaussow (@xoxys) - Javier Ferrer (@javfg) - Vasco Guita (@vascoguita) -- Tomas Zaluckij (@Tomaszal) Initial revision: December 2016
First production version for CERNBox: September 2017 (presented at [oCCon17](https://occon17.owncloud.org) - [slides](https://www.slideshare.net/giuseppelopresti/collaborative-editing-and-more-in-cernbox))
diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 42ff5b89..3d384332 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -54,6 +54,7 @@ class ViewMode(Enum): # The file can be downloaded and updated, and the app should be shown in preview mode PREVIEW = "VIEW_MODE_PREVIEW" + class UserType(Enum): '''App user types as given by https://github.com/cs3org/reva/blob/master/pkg/app/provider/wopi/wopi.go From 7a47a6c6810f935b95412c1e7cc33937d7508a96 Mon Sep 17 00:00:00 2001 From: Vasco Guita Date: Fri, 10 Mar 2023 11:10:39 +0100 Subject: [PATCH 187/325] Add --amend option to docker manifest --- .github/workflows/release.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index adfc6563..d67e15a4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,21 +53,26 @@ jobs: VERSION=${{ github.ref_name }} BASEIMAGE=${{ matrix.image }} platforms: ${{ matrix.platform }} - build-multi-architecture: + manifest: runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'cs3org/wopiserver'] }} needs: release if: github.event_name != 'workflow_dispatch' + strategy: + fail-fast: false + matrix: + manifest: + - ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }} + - ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:latest steps: - name: Login to Docker Hub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - uses: int128/docker-manifest-create-action@v1 - with: - tags: | - ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }} - ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:latest - suffixes: | - -amd64 - -arm64 \ No newline at end of file + - name: Create manifest + run: | + docker manifest create ${{ matrix.manifest }} \ + --amend ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-amd64 \ + --amend ${{ vars.DOCKERHUB_ORGANIZATION }}/wopiserver:${{ github.ref_name }}-arm64 + - name: Push manifest + run: docker manifest push ${{ matrix.manifest }} \ No newline at end of file From 33e92151d6fb879b78b016baf1b7e9358d60237a Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 10 Mar 2023 11:45:04 +0100 Subject: [PATCH 188/325] Add usertype to wopiopen --- tools/wopiopen.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/wopiopen.py b/tools/wopiopen.py index 44c8eeeb..99de7021 100755 --- a/tools/wopiopen.py +++ b/tools/wopiopen.py @@ -13,7 +13,7 @@ import configparser import requests sys.path.append('src') # for tests out of the git repo -from core.wopiutils import ViewMode # noqa: E402 +from core.wopiutils import ViewMode, UserType # noqa: E402 # usage function @@ -21,18 +21,20 @@ def usage(exitcode): '''Prints usage''' print('Usage : ' + sys.argv[0] + ' -a|--appname -u|--appurl [-i|--appinturl ] ' '-k|--apikey [-s|--storage ] [-v|--viewmode VIEW_ONLY|READ_ONLY|READ_WRITE|PREVIEW] ' - '[-x|--x-access-token ] ') + '[-t|--user-type REGULAR|FEDERATED|ANONYMOUS] [-x|--x-access-token ] ') sys.exit(exitcode) # first parse the options try: - options, args = getopt.getopt(sys.argv[1:], 'hv:s:a:i:u:x:k:', - ['help', 'viewmode', 'storage', 'appname', 'appinturl', 'appurl', 'x-access-token', 'apikey']) + options, args = getopt.getopt(sys.argv[1:], 'hv:t:s:a:i:u:x:k:', + ['help', 'viewmode', 'usertype', 'storage', 'appname', 'appinturl', 'appurl', + 'x-access-token', 'apikey']) except getopt.GetoptError as e: print(e) usage(1) viewmode = ViewMode.READ_WRITE +usertype = UserType.REGULAR endpoint = '' appname = '' appurl = '' @@ -49,6 +51,12 @@ def usage(exitcode): except ValueError: print("Invalid argument for viewmode: " + v) usage(1) + elif f == '-t' or f == '--usertype': + try: + usertype = UserType(v) + except ValueError: + print("Invalid argument for usertype: " + v) + usage(1) elif f == '-s' or f == '--storage': endpoint = v elif f == '-i' or f == '--appinturl': @@ -111,8 +119,8 @@ def usage(exitcode): # open the file and get WOPI token wopiheaders = {'Authorization': 'Bearer ' + iopsecret} -wopiparams = {'fileid': filename, 'endpoint': endpoint, - 'viewmode': viewmode.value, 'username': 'Operator', 'userid': userid, 'folderurl': '/', +wopiparams = {'fileid': filename, 'endpoint': endpoint, 'viewmode': viewmode.value, 'usertype': usertype.value, + 'username': 'Operator', 'userid': userid, 'folderurl': '/', 'appurl': appurl, 'appinturl': appinturl, 'appname': appname} wopiheaders['TokenHeader'] = revatoken # for bridged apps, also set the API key From 53af13b5051488a4213026e83b34b516eca81d8b Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 14 Mar 2023 12:42:38 +0100 Subject: [PATCH 189/325] xroot: fail statx in case of errors on the version folder --- src/core/xrootiface.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index c7976df8..11cf8bd3 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -261,9 +261,8 @@ def statx(endpoint, fileref, userid, versioninv=1): log.debug('msg="Invoked stat on version folder" endpoint="%s" filepath="%s" rc="%s" result="%s" elapsedTimems="%.1f"' % (endpoint, _getfilepath(verFolder), str(rcv).strip('\n'), infov, (tend-tstart)*1000)) except IOError as e: - # here we should really raise the error, but for now we just log it - log.error('msg="Failed to mkdir/stat version folder, returning file metadata instead" filepath="%s" error="%s"' % - (_getfilepath(filepath), e)) + log.error('msg="Failed to mkdir/stat version folder" filepath="%s" error="%s"' % (_getfilepath(filepath), e)) + raise # return the metadata of the given file, with the inode taken from the version folder endpoint = _geturlfor(endpoint) inode = common.encodeinode(endpoint[7:] if endpoint.find('.') == -1 else endpoint[7:endpoint.find('.')], statxdata['ino']) From 7a7f6ec59322a8c95652e4e9ebaff96ca60783e7 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 15 Mar 2023 10:16:43 +0100 Subject: [PATCH 190/325] Fixed comments --- src/core/cs3iface.py | 2 +- src/core/localiface.py | 5 ++--- src/core/xrootiface.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index 2fef99e8..0ab9586d 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -267,7 +267,7 @@ def unlock(endpoint, filepath, userid, appname, value): def readfile(endpoint, filepath, userid, lockid): - '''Read a file using the given userid as access token. Note that the function is a generator, managed by Flask.''' + '''Read a file using the given userid as access token. Note that the function is a generator, managed by the app server.''' tstart = time.time() reference = _getcs3reference(endpoint, filepath) diff --git a/src/core/localiface.py b/src/core/localiface.py index 10f848ef..d5b6bfb7 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -207,7 +207,7 @@ def unlock(endpoint, filepath, userid, appname, value): def readfile(_endpoint, filepath, _userid, _lockid): - '''Read a file on behalf of the given userid. Note that the function is a generator, managed by Flask.''' + '''Read a file on behalf of the given userid. Note that the function is a generator, managed by the app server.''' log.debug('msg="Invoking readFile" filepath="%s"' % filepath) try: tstart = time.time() @@ -215,7 +215,7 @@ def readfile(_endpoint, filepath, _userid, _lockid): with open(_getfilepath(filepath), mode='rb', buffering=chunksize) as f: tend = time.time() log.info('msg="File open for read" filepath="%s" elapsedTimems="%.1f"' % (filepath, (tend - tstart) * 1000)) - # the actual read is buffered and managed by the Flask server + # the actual read is buffered and managed by the app server for chunk in iter(lambda: f.read(chunksize), b''): yield chunk except FileNotFoundError: @@ -224,7 +224,6 @@ def readfile(_endpoint, filepath, _userid, _lockid): # as this is a generator, we yield the error string instead of the file's contents yield IOError('No such file or directory') except OSError as e: - # general case, issue a warning log.error('msg="Error opening the file for read" filepath="%s" error="%s"' % (filepath, e)) yield IOError(e) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index 11cf8bd3..beaf3b77 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -408,7 +408,7 @@ def readfile(endpoint, filepath, userid, _lockid): chunksize = config.getint('io', 'chunksize') rc, statInfo = f.stat() chunksize = min(chunksize, statInfo.size) - # the actual read is buffered and managed by the Flask server + # the actual read is buffered and managed by the application server for chunk in f.readchunks(offset=0, chunksize=chunksize): yield chunk From 596b89f72e6b747f20555c5b759fa80932f7b49b Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 16 Mar 2023 14:20:18 +0100 Subject: [PATCH 191/325] Improved logging --- src/core/wopiutils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index 3d384332..efd3c38b 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -252,9 +252,9 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app acctok = jwt.encode(tokmd, srv.wopisecret, algorithm='HS256') if 'MS 365' in appname: srv.allusers.add(userid) - log.info('msg="Access token generated" userid="%s" wopiuser="%s" friendlyname="%s" mode="%s" endpoint="%s" filename="%s" ' - 'inode="%s" mtime="%s" folderurl="%s" appname="%s"%s expiration="%d" token="%s"' % - (userid[-20:], wopiuser, friendlyname, viewmode, endpoint, statinfo['filepath'], statinfo['inode'], + log.info('msg="Access token generated" userid="%s" wopiuser="%s" friendlyname="%s" usertype="%s" mode="%s" ' + 'endpoint="%s" filename="%s" inode="%s" mtime="%s" folderurl="%s" appname="%s"%s expiration="%d" token="%s"' % + (userid[-20:], wopiuser, friendlyname, usertype, viewmode, endpoint, statinfo['filepath'], statinfo['inode'], statinfo['mtime'], folderurl, appname, ' forcelock="True"' if forcelock else '', exptime, acctok[-20:])) return statinfo['inode'], acctok, viewmode From 3552bfdf7a6d9bcedac1e6dc2ea040e1de4e5ab9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 22 Mar 2023 10:43:10 +0100 Subject: [PATCH 192/325] Added backwards compatibility code, to be reverted --- src/core/wopi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index fef1461f..61f0d2df 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -58,21 +58,21 @@ def checkFileInfo(fileid, acctok): furl = acctok['folderurl'] if furl != '/': fmd['BreadcrumbFolderUrl'] = furl + '?scrollTo=' + fmd['BaseFileName'] - if acctok['username'] == '' or acctok['usertype'] == utils.UserType.ANONYMOUS: + if acctok['username'] == '' or 'usertype' in acctok and acctok['usertype'] == utils.UserType.ANONYMOUS: fmd['IsAnonymousUser'] = True fmd['UserFriendlyName'] = 'Guest ' + utils.randomString(3) fmd['BreadcrumbFolderName'] = 'Public share' else: fmd['IsAnonymousUser'] = False fmd['UserFriendlyName'] = acctok['username'] - fmd['BreadcrumbFolderName'] = 'ScienceMesh share' if acctok['usertype'] == utils.UserType.OCM else 'Parent folder' + fmd['BreadcrumbFolderName'] = 'ScienceMesh share' if 'usertype' in acctok and acctok['usertype'] == utils.UserType.OCM else 'Parent folder' if acctok['viewmode'] in (utils.ViewMode.READ_ONLY, utils.ViewMode.READ_WRITE) \ and srv.config.get('general', 'downloadurl', fallback=None): fmd['DownloadUrl'] = fmd['FileUrl'] = '%s?access_token=%s' % \ (srv.config.get('general', 'downloadurl'), flask.request.args['access_token']) if srv.config.get('general', 'businessflow', fallback='False').upper() == 'TRUE': # enable the check for real users, not for public links / federated access - fmd['LicenseCheckForEditIsEnabled'] = acctok['usertype'] == utils.UserType.REGULAR + fmd['LicenseCheckForEditIsEnabled'] = 'usertype' not in acctok or acctok['usertype'] == utils.UserType.REGULAR fmd['BreadcrumbBrandName'] = srv.config.get('general', 'brandingname', fallback=None) fmd['BreadcrumbBrandUrl'] = srv.config.get('general', 'brandingurl', fallback=None) fmd['OwnerId'] = statInfo['ownerid'] @@ -87,7 +87,7 @@ def checkFileInfo(fileid, acctok): # no personal space where to save as an alternate location. # Note that single-file r/w shares are optimistically offered a SaveAs option, which may only work for regular users. fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE and \ - acctok['usertype'] != utils.UserType.REGULAR + ('usertype' not in acctok or acctok['usertype'] != utils.UserType.REGULAR) fmd['SupportsRename'] = fmd['UserCanRename'] = enablerename and (acctok['viewmode'] == utils.ViewMode.READ_WRITE) fmd['SupportsContainers'] = False # TODO this is all to be implemented fmd['SupportsUserInfo'] = True @@ -426,7 +426,7 @@ def putRelative(fileid, reqheaders, acctok): return IO_ERROR, http.client.INTERNAL_SERVER_ERROR raisenoaccess = True # make an attempt in the user's home if possible - if acctok['usertype'] == utils.UserType.REGULAR: + if 'usertype' in acctok and acctok['usertype'] == utils.UserType.REGULAR: targetName = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ replace('username', acctok['wopiuser'].split('!')[0]) \ + os.path.sep + os.path.basename(targetName) # noqa: E131 From 90fa2e4d31bfbb39a4705df59681f52a33b19929 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 24 Mar 2023 18:24:08 +0100 Subject: [PATCH 193/325] xroot: catch unicode decode errors as they have happened in production --- src/core/xrootiface.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/core/xrootiface.py b/src/core/xrootiface.py index beaf3b77..853565ee 100644 --- a/src/core/xrootiface.py +++ b/src/core/xrootiface.py @@ -115,7 +115,12 @@ def _xrootcmd(endpoint, cmd, subcmd, userid, args, app='wopi'): if not f.is_open(): log.error('msg="Error or timeout with xroot" cmd="%s" subcmd="%s" args="%s" rc="%s"' % (cmd, subcmd, args, rc)) raise IOError('Timeout executing %s' % cmd) - res = b''.join(f.readlines()).decode().split('&') + res = b''.join(f.readlines()) + try: + res = res.decode().split('&') + except UnicodeDecodeError as e: + log.error('msg="Failed to decode cmd output" cmd="%s" subcmd="%s" args="%s" res="%s" error="%s"' % (cmd, subcmd, args, res, e)) + raise IOError('Failed to decode cmd output') if len(res) == 3: # we may only just get stdout: in that case, assume it's all OK rc = res[2].strip('\n') rc = rc[rc.find('=') + 1:].strip('\00') @@ -260,9 +265,9 @@ def statx(endpoint, fileref, userid, versioninv=1): statxdata['ino'] = infov.split()[2] log.debug('msg="Invoked stat on version folder" endpoint="%s" filepath="%s" rc="%s" result="%s" elapsedTimems="%.1f"' % (endpoint, _getfilepath(verFolder), str(rcv).strip('\n'), infov, (tend-tstart)*1000)) - except IOError as e: + except (IOError, UnicodeDecodeError) as e: log.error('msg="Failed to mkdir/stat version folder" filepath="%s" error="%s"' % (_getfilepath(filepath), e)) - raise + raise IOError(e) # return the metadata of the given file, with the inode taken from the version folder endpoint = _geturlfor(endpoint) inode = common.encodeinode(endpoint[7:] if endpoint.find('.') == -1 else endpoint[7:endpoint.find('.')], statxdata['ino']) From 33c7ca671fdda923f256b4104d43064c0972e975 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 24 Mar 2023 18:40:51 +0100 Subject: [PATCH 194/325] Minor rephrasing --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68bde8fa..017f5c63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ folder URLs and SaveAs operations - Redefined `conflictpath` option as `homepath` (the former is still supported for backwards compatibility): when defined, - a SaveAs operation falls back to the user's home when it - can't work on the original folder + a SaveAs operation falls back to the user's `homepath` when + it can't work on the original folder - Fixed PutUserInfo to use the user's username as xattr key - Added arm64-based builds From a04ddaae6a8bfcefb00bddb23b9f47981e87ed28 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 25 Mar 2023 16:31:58 +0100 Subject: [PATCH 195/325] Fixed missing property --- src/core/wopi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 61f0d2df..27bc03f2 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -447,7 +447,7 @@ def putRelative(fileid, reqheaders, acctok): 'mode="ViewMode.READ_WRITE" friendlyname="%s"' % (acctok['userid'][-20:], targetName, acctok['username'])) inode, newacctok, _ = utils.generateAccessToken(acctok['userid'], targetName, utils.ViewMode.READ_WRITE, - (acctok['username'], acctok['wopiuser']), + (acctok['username'], acctok['wopiuser'], utils.UserType(acctok['usertype']) if 'usertype' in acctok else utils.UserType.REGULAR), acctok['folderurl'], acctok['endpoint'], (acctok['appname'], acctok['appediturl'], acctok['appviewurl'])) # prepare and send the response as JSON From 902e7d0f98383480fc58ff4221543b06f3da5ea1 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 25 Mar 2023 16:51:13 +0100 Subject: [PATCH 196/325] Revert "Added backwards compatibility code, to be reverted" This reverts commit 3552bfdf7a6d9bcedac1e6dc2ea040e1de4e5ab9. --- src/core/wopi.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/wopi.py b/src/core/wopi.py index 27bc03f2..37a7b590 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -58,21 +58,21 @@ def checkFileInfo(fileid, acctok): furl = acctok['folderurl'] if furl != '/': fmd['BreadcrumbFolderUrl'] = furl + '?scrollTo=' + fmd['BaseFileName'] - if acctok['username'] == '' or 'usertype' in acctok and acctok['usertype'] == utils.UserType.ANONYMOUS: + if acctok['username'] == '' or acctok['usertype'] == utils.UserType.ANONYMOUS: fmd['IsAnonymousUser'] = True fmd['UserFriendlyName'] = 'Guest ' + utils.randomString(3) fmd['BreadcrumbFolderName'] = 'Public share' else: fmd['IsAnonymousUser'] = False fmd['UserFriendlyName'] = acctok['username'] - fmd['BreadcrumbFolderName'] = 'ScienceMesh share' if 'usertype' in acctok and acctok['usertype'] == utils.UserType.OCM else 'Parent folder' + fmd['BreadcrumbFolderName'] = 'ScienceMesh share' if acctok['usertype'] == utils.UserType.OCM else 'Parent folder' if acctok['viewmode'] in (utils.ViewMode.READ_ONLY, utils.ViewMode.READ_WRITE) \ and srv.config.get('general', 'downloadurl', fallback=None): fmd['DownloadUrl'] = fmd['FileUrl'] = '%s?access_token=%s' % \ (srv.config.get('general', 'downloadurl'), flask.request.args['access_token']) if srv.config.get('general', 'businessflow', fallback='False').upper() == 'TRUE': # enable the check for real users, not for public links / federated access - fmd['LicenseCheckForEditIsEnabled'] = 'usertype' not in acctok or acctok['usertype'] == utils.UserType.REGULAR + fmd['LicenseCheckForEditIsEnabled'] = acctok['usertype'] == utils.UserType.REGULAR fmd['BreadcrumbBrandName'] = srv.config.get('general', 'brandingname', fallback=None) fmd['BreadcrumbBrandUrl'] = srv.config.get('general', 'brandingurl', fallback=None) fmd['OwnerId'] = statInfo['ownerid'] @@ -87,7 +87,7 @@ def checkFileInfo(fileid, acctok): # no personal space where to save as an alternate location. # Note that single-file r/w shares are optimistically offered a SaveAs option, which may only work for regular users. fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE and \ - ('usertype' not in acctok or acctok['usertype'] != utils.UserType.REGULAR) + acctok['usertype'] != utils.UserType.REGULAR fmd['SupportsRename'] = fmd['UserCanRename'] = enablerename and (acctok['viewmode'] == utils.ViewMode.READ_WRITE) fmd['SupportsContainers'] = False # TODO this is all to be implemented fmd['SupportsUserInfo'] = True @@ -426,7 +426,7 @@ def putRelative(fileid, reqheaders, acctok): return IO_ERROR, http.client.INTERNAL_SERVER_ERROR raisenoaccess = True # make an attempt in the user's home if possible - if 'usertype' in acctok and acctok['usertype'] == utils.UserType.REGULAR: + if acctok['usertype'] == utils.UserType.REGULAR: targetName = srv.homepath.replace('user_initial', acctok['wopiuser'][0]). \ replace('username', acctok['wopiuser'].split('!')[0]) \ + os.path.sep + os.path.basename(targetName) # noqa: E131 @@ -447,7 +447,7 @@ def putRelative(fileid, reqheaders, acctok): 'mode="ViewMode.READ_WRITE" friendlyname="%s"' % (acctok['userid'][-20:], targetName, acctok['username'])) inode, newacctok, _ = utils.generateAccessToken(acctok['userid'], targetName, utils.ViewMode.READ_WRITE, - (acctok['username'], acctok['wopiuser'], utils.UserType(acctok['usertype']) if 'usertype' in acctok else utils.UserType.REGULAR), + (acctok['username'], acctok['wopiuser'], utils.UserType(acctok['usertype'])), acctok['folderurl'], acctok['endpoint'], (acctok['appname'], acctok['appediturl'], acctok['appviewurl'])) # prepare and send the response as JSON From ac963046c19f3b0013e8071372e824fa9101f6ec Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Sat, 25 Mar 2023 17:00:24 +0100 Subject: [PATCH 197/325] Use f-strings as opposed to "%" (converted via flynt) --- src/bridge/__init__.py | 32 +++++++++++++------------- src/bridge/codimd.py | 32 +++++++++++++------------- src/bridge/etherpad.py | 20 ++++++++-------- src/bridge/wopiclient.py | 22 +++++++++--------- src/core/cs3iface.py | 38 +++++++++++++++---------------- src/core/localiface.py | 38 +++++++++++++++---------------- src/core/wopi.py | 36 ++++++++++++++--------------- src/core/wopiutils.py | 28 +++++++++++------------ src/core/xrootiface.py | 48 +++++++++++++++++++-------------------- src/wopiserver.py | 39 +++++++++++++++---------------- test/test_storageiface.py | 14 ++++++------ 11 files changed, 174 insertions(+), 173 deletions(-) diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index a20d7985..ba5f7f63 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -104,7 +104,7 @@ def loadplugin(cls, appname, appurl, appinturl, apikey): cls.plugins[p].disablezip = cls.disablezip cls.plugins[p].appname = appname cls.plugins[p].init(appurl, appinturl, apikey) - cls.log.info('msg="Imported plugin for application" app="%s" plugin="%s"' % (p, cls.plugins[p])) + cls.log.info(f'msg="Imported plugin for application" app="{p}" plugin="{cls.plugins[p]}"') except Exception as e: cls.log.warning('msg="Failed to initialize plugin" app="%s" URL="%s" exception="%s"' % (p, appinturl, e)) @@ -133,7 +133,7 @@ def _validateappname(appname): for p in WB.plugins.values(): if appname.lower() in p.appname.lower(): return p.appname - WB.log.debug('msg="BridgeSave: unknown application" appname="%s" plugins="%s"' % (appname, WB.plugins.values())) + WB.log.debug(f'msg="BridgeSave: unknown application" appname="{appname}" plugins="{WB.plugins.values()}"') raise ValueError @@ -160,13 +160,13 @@ def appopen(wopisrc, acctok, appmd, viewmode, revatok=None): WB.loadplugin(appname, appurl, appinturl, apikey) appname = _validateappname(appname) app = WB.plugins[appname] - WB.log.debug('msg="BridgeOpen: processing supported app" appname="%s" plugin="%s"' % (appname, app)) + WB.log.debug(f'msg="BridgeOpen: processing supported app" appname="{appname}" plugin="{app}"') except ValueError: WB.log.warning('msg="BridgeOpen: appname not supported or missing plugin" appname="%s" token="%s"' % (appname, acctok[-20:])) - raise FailedOpen('Failed to load WOPI bridge plugin for %s' % appname, http.client.INTERNAL_SERVER_ERROR) + raise FailedOpen(f'Failed to load WOPI bridge plugin for {appname}', http.client.INTERNAL_SERVER_ERROR) except KeyError: - raise FailedOpen('Bridged app %s already configured with a different appurl' % appname, http.client.NOT_IMPLEMENTED) + raise FailedOpen(f'Bridged app {appname} already configured with a different appurl', http.client.NOT_IMPLEMENTED) # WOPI GetFileInfo res = wopic.request(wopisrc, acctok, 'GET') @@ -181,14 +181,14 @@ def appopen(wopisrc, acctok, appmd, viewmode, revatok=None): try: # was it already being worked on? wopilock = wopic.getlock(wopisrc, acctok) - WB.log.info('msg="Lock already held" lock="%s" token="%s"' % (wopilock, acctok[-20:])) + WB.log.info(f'msg="Lock already held" lock="{wopilock}" token="{acctok[-20:]}"') # add this token to the list, if not already in if acctok[-20:] not in wopilock['tocl']: wopilock = wopic.refreshlock(wopisrc, acctok, wopilock) except wopic.InvalidLock as e: if str(e) != str(int(http.client.NOT_FOUND)): # lock is invalid/corrupted: force read-only mode - WB.log.info('msg="Invalid lock, forcing read-only mode" error="%s" token="%s"' % (e, acctok[-20:])) + WB.log.info(f'msg="Invalid lock, forcing read-only mode" error="{e}" token="{acctok[-20:]}"') filemd['UserCanWrite'] = False # otherwise, this is the first user opening the file; in both cases, fetch it @@ -270,11 +270,11 @@ def appsave(docid): except KeyError as e: WB.log.error('msg="BridgeSave: missing metadata" address="%s" headers="%s" args="%s" error="%s"' % (flask.request.remote_addr, flask.request.headers, flask.request.args, e)) - return wopic.jsonify('Missing metadata, could not save. %s' % RECOVER_MSG), http.client.BAD_REQUEST + return wopic.jsonify(f'Missing metadata, could not save. {RECOVER_MSG}'), http.client.BAD_REQUEST except ValueError: WB.log.error('msg="BridgeSave: unknown application" address="%s" appheader="%s" args="%s"' % (flask.request.remote_addr, flask.request.headers.get(BRIDGED_APPNAME_HEADER), flask.request.args)) - return wopic.jsonify('Unknown application, could not save. %s' % RECOVER_MSG), http.client.BAD_REQUEST + return wopic.jsonify(f'Unknown application, could not save. {RECOVER_MSG}'), http.client.BAD_REQUEST # decide whether to notify the save thread donotify = isclose or wopisrc not in WB.openfiles or WB.openfiles[wopisrc]['lastsave'] < time.time() - WB.saveinterval @@ -284,7 +284,7 @@ def appsave(docid): WB.openfiles[wopisrc]['tosave'] = True WB.openfiles[wopisrc]['toclose'][acctok[-20:]] = isclose else: - WB.log.info('msg="Save: repopulating missing metadata" wopisrc="%s" token="%s"' % (wopisrc, acctok[-20:])) + WB.log.info(f'msg="Save: repopulating missing metadata" wopisrc="{wopisrc}" token="{acctok[-20:]}"') WB.openfiles[wopisrc] = { 'acctok': acctok, 'tosave': True, 'lastsave': int(time.time() - WB.saveinterval), @@ -308,10 +308,10 @@ def appsave(docid): logf = WB.log.error else: logf = WB.log.info - logf('msg="BridgeSave: returned response" response="%s" token="%s"' % (resp, acctok[-20:])) + logf(f'msg="BridgeSave: returned response" response="{resp}" token="{acctok[-20:]}"') del WB.saveresponses[wopisrc] return resp - WB.log.info('msg="BridgeSave: enqueued action" immediate="%s" token="%s"' % (donotify, acctok[-20:])) + WB.log.info(f'msg="BridgeSave: enqueued action" immediate="{donotify}" token="{acctok[-20:]}"') return '{}', http.client.ACCEPTED @@ -322,7 +322,7 @@ def applist(): WB.log.warning('msg="BridgeList: unauthorized access attempt, missing authorization token" ' 'client="%s"' % flask.request.remote_addr) return 'Client not authorized', http.client.UNAUTHORIZED - WB.log.info('msg="BridgeList: returning list of open files" client="%s"' % flask.request.remote_addr) + WB.log.info(f'msg="BridgeList: returning list of open files" client="{flask.request.remote_addr}"') return flask.Response(json.dumps(WB.openfiles), mimetype='application/json') @@ -440,7 +440,7 @@ def closewhenidle(self, openfile, wopisrc, wopilock): (openfile['lastsave'], openfile['toclose'])) except wopic.InvalidLock: # lock is gone, just cleanup our metadata - WB.log.warning('msg="SaveThread: cleaning up metadata, detected missed close event" url="%s"' % wopisrc) + WB.log.warning(f'msg="SaveThread: cleaning up metadata, detected missed close event" url="{wopisrc}"') del WB.openfiles[wopisrc] return wopilock @@ -454,7 +454,7 @@ def cleanup(self, openfile, wopisrc, wopilock): # nothing to do here, this document may have been closed by another wopibridge if openfile['lastsave'] < time.time() - WB.unlockinterval: # yet clean up only after the unlockinterval time, cf. the InvalidLock handling in savedirty() - WB.log.info('msg="SaveThread: cleaning up metadata, file already unlocked" url="%s"' % wopisrc) + WB.log.info(f'msg="SaveThread: cleaning up metadata, file already unlocked" url="{wopisrc}"') try: del WB.openfiles[wopisrc] except KeyError: @@ -482,7 +482,7 @@ def cleanup(self, openfile, wopisrc, wopilock): try: wopic.refreshlock(wopisrc, openfile['acctok'], wopilock, toclose=openfile['toclose']) except wopic.InvalidLock: - WB.log.warning('msg="SaveThread: failed to refresh lock, will retry" url="%s"' % wopisrc) + WB.log.warning(f'msg="SaveThread: failed to refresh lock, will retry" url="{wopisrc}"') @atexit.register diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index ee109983..9901e277 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -45,11 +45,11 @@ def init(_appurl, _appinturl, _apikey): # CodiMD integrates Prometheus metrics, let's probe if they exist res = requests.head(appurl + '/metrics/codimd', verify=sslverify) if res.status_code != http.client.OK: - log.error('msg="The provided URL does not seem to be a CodiMD instance" appurl="%s"' % appurl) + log.error(f'msg="The provided URL does not seem to be a CodiMD instance" appurl="{appurl}"') raise AppFailure - log.info('msg="Successfully connected to CodiMD" appurl="%s"' % appurl) + log.info(f'msg="Successfully connected to CodiMD" appurl="{appurl}"') except requests.exceptions.ConnectionError as e: - log.error('msg="Exception raised attempting to connect to CodiMD" exception="%s"' % e) + log.error(f'msg="Exception raised attempting to connect to CodiMD" exception="{e}"') raise AppFailure @@ -81,7 +81,7 @@ def _unzipattachments(inputbuf): mddoc = None for zipinfo in inputzip.infolist(): fname = zipinfo.filename - log.debug('msg="Extracting attachment" name="%s"' % fname) + log.debug(f'msg="Extracting attachment" name="{fname}"') if os.path.splitext(fname)[1] == '.md': mddoc = inputzip.read(zipinfo) else: @@ -89,18 +89,18 @@ def _unzipattachments(inputbuf): res = requests.head(appurl + '/uploads/' + fname, verify=sslverify) if res.status_code == http.client.OK and int(res.headers['Content-Length']) == zipinfo.file_size: # yes (assume that hashed filename AND size matching is a good enough content match!) - log.debug('msg="Skipped existing attachment" filename="%s"' % fname) + log.debug(f'msg="Skipped existing attachment" filename="{fname}"') continue # check for collision if res.status_code == http.client.OK: - log.warning('msg="Attachment collision detected" filename="%s"' % fname) + log.warning(f'msg="Attachment collision detected" filename="{fname}"') # append a random letter to the filename name, ext = os.path.splitext(fname) fname = name + '_' + chr(randint(65, 65+26)) + ext # and replace its reference in the document (this creates a copy of the doc, not very efficient) mddoc = mddoc.replace(bytes(zipinfo.filename), bytes(fname)) # OK, let's upload - log.debug('msg="Pushing attachment" filename="%s"' % fname) + log.debug(f'msg="Pushing attachment" filename="{fname}"') res = requests.post(appurl + '/uploadimage', params={'generateFilename': 'false'}, files={'image': (fname, inputzip.read(zipinfo))}, verify=sslverify) if res.status_code != http.client.OK: @@ -121,7 +121,7 @@ def _fetchfromcodimd(wopilock, acctok): raise AppFailure return res.content except requests.exceptions.ConnectionError as e: - log.error('msg="Exception raised attempting to connect to CodiMD" exception="%s"' % e) + log.error(f'msg="Exception raised attempting to connect to CodiMD" exception="{e}"') raise AppFailure @@ -148,14 +148,14 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): headers={'Content-Type': 'text/markdown'}, verify=sslverify) if res.status_code == http.client.REQUEST_ENTITY_TOO_LARGE: - log.error('msg="File is too large to be edited in CodiMD" token="%s"' % acctok[-20:]) + log.error(f'msg="File is too large to be edited in CodiMD" token="{acctok[-20:]}"') raise AppFailure(TOOLARGE) if res.status_code != http.client.FOUND: log.error('msg="Unable to push read-only document to CodiMD" token="%s" response="%d"' % (acctok[-20:], res.status_code)) raise AppFailure docid = urlparse.urlsplit(res.headers['location']).path.split('/')[-1] - log.info('msg="Pushed read-only document to CodiMD" docid="%s" token="%s"' % (docid, acctok[-20:])) + log.info(f'msg="Pushed read-only document to CodiMD" docid="{docid}" token="{acctok[-20:]}"') else: # reserve the given docid in CodiMD via a HEAD request @@ -180,21 +180,21 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): verify=sslverify) if res.status_code == http.client.FORBIDDEN: # the file got unlocked because of no activity, yet some user is there: let it go - log.warning('msg="Document was being edited in CodiMD, redirecting user" token="%s"' % acctok[-20:]) + log.warning(f'msg="Document was being edited in CodiMD, redirecting user" token="{acctok[-20:]}"') elif res.status_code == http.client.REQUEST_ENTITY_TOO_LARGE: - log.error('msg="File is too large to be edited in CodiMD" docid="%s" token="%s"' % (docid, acctok[-20:])) + log.error(f'msg="File is too large to be edited in CodiMD" docid="{docid}" token="{acctok[-20:]}"') raise AppFailure(TOOLARGE) elif res.status_code != http.client.OK: log.error('msg="Unable to push document to CodiMD" docid="%s" token="%s" response="%d"' % (docid, acctok[-20:], res.status_code)) raise AppFailure - log.info('msg="Pushed document to CodiMD" docid="%s" token="%s"' % (docid, acctok[-20:])) + log.info(f'msg="Pushed document to CodiMD" docid="{docid}" token="{acctok[-20:]}"') except requests.exceptions.ConnectionError as e: - log.error('msg="Exception raised attempting to connect to CodiMD" exception="%s"' % e) + log.error(f'msg="Exception raised attempting to connect to CodiMD" exception="{e}"') raise AppFailure except UnicodeDecodeError as e: - log.warning('msg="Invalid UTF-8 content found in file" exception="%s"' % e) + log.warning(f'msg="Invalid UTF-8 content found in file" exception="{e}"') raise AppFailure('File contains an invalid UTF-8 character, was it corrupted? ' + 'Please fix it in a regular editor before opening it in CodiMD.') # generate and return a WOPI lock structure for this document @@ -209,7 +209,7 @@ def _getattachments(mddoc, docfilename, forcezip=False): zip_buffer = io.BytesIO() response = None for attachment in upload_re.findall(mddoc): - log.debug('msg="Fetching attachment" url="%s"' % attachment) + log.debug(f'msg="Fetching attachment" url="{attachment}"') res = requests.get(appurl + attachment, verify=sslverify) if res.status_code != http.client.OK: log.error('msg="Failed to fetch included file, skipping" path="%s" response="%d"' % ( diff --git a/src/bridge/etherpad.py b/src/bridge/etherpad.py index 837b724f..403e9396 100644 --- a/src/bridge/etherpad.py +++ b/src/bridge/etherpad.py @@ -40,7 +40,7 @@ def init(_appurl, _appinturl, _apikey): # create a general group to attach all pads; can raise AppFailure groupid = _apicall('createGroupIfNotExistsFor', {'groupMapper': 1}) groupid = groupid['data']['groupID'] - log.info('msg="Got Etherpad global groupid" groupid="%s"' % groupid) + log.info(f'msg="Got Etherpad global groupid" groupid="{groupid}"') def _apicall(method, params, data=None, acctok=None, raiseonnonzerocode=True): @@ -53,7 +53,7 @@ def _apicall(method, params, data=None, acctok=None, raiseonnonzerocode=True): (method, acctok[-20:] if acctok else 'N/A', res.status_code, res.content.decode())) raise AppFailure except requests.exceptions.ConnectionError as e: - log.error('msg="Exception raised attempting to connect to Etherpad" method="%s" exception="%s"' % (method, e)) + log.error(f'msg="Exception raised attempting to connect to Etherpad" method="{method}" exception="{e}"') raise AppFailure res = res.json() if res['code'] != 0 and raiseonnonzerocode: @@ -70,7 +70,7 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, _filename, displayname, _re if viewmode in (utils.ViewMode.READ_ONLY, utils.ViewMode.VIEW_ONLY): # for read-only mode generate a read-only link res = _apicall('getReadOnlyID', {'padID': docid}, acctok=acctok) - return appexturl + '/p/%s?userName=%s' % (res['data']['readOnlyID'], urlparse.quote_plus(displayname)) + return appexturl + f"/p/{res['data']['readOnlyID']}?userName={urlparse.quote_plus(displayname)}" # pass to Etherpad the required metadata for the save webhook try: @@ -82,13 +82,13 @@ def getredirecturl(viewmode, wopisrc, acctok, docid, _filename, displayname, _re log.error('msg="Failed to call Etherpad" method="setEFSSMetadata" token="%s" response="%d: %s"' % (acctok[-20:], res.status_code, res.content.decode().replace('"', "'"))) raise AppFailure - log.debug('msg="Called Etherpad" method="setEFSSMetadata" token="%s"' % acctok[-20:]) + log.debug(f'msg="Called Etherpad" method="setEFSSMetadata" token="{acctok[-20:]}"') except requests.exceptions.ConnectionError as e: - log.error('msg="Exception raised attempting to connect to Etherpad" method="setEFSSMetadata" exception="%s"' % e) + log.error(f'msg="Exception raised attempting to connect to Etherpad" method="setEFSSMetadata" exception="{e}"') raise AppFailure # return the URL to the pad for editing (a PREVIEW viewmode is not supported) - return appexturl + '/p/%s?userName=%s' % (docid, urlparse.quote_plus(displayname)) + return appexturl + f'/p/{docid}?userName={urlparse.quote_plus(displayname)}' # Cloud storage to Etherpad @@ -106,7 +106,7 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): try: if not docid: docid = ''.join([choice(ascii_lowercase) for _ in range(20)]) - log.debug('msg="Generated random padID for read-only document" padid="%s" token="%s"' % (docid, acctok[-20:])) + log.debug(f'msg="Generated random padID for read-only document" padid="{docid}" token="{acctok[-20:]}"') # first drop previous pad if it exists _apicall('deletePad', {'padID': docid}, acctok=acctok, raiseonnonzerocode=False) # create pad with the given docid as name @@ -121,9 +121,9 @@ def loadfromstorage(filemd, wopisrc, acctok, docid): log.error('msg="Unable to push document to Etherpad" token="%s" padid="%s" response="%d: %s"' % (acctok[-20:], docid, res.status_code, res.content.decode())) raise AppFailure - log.info('msg="Pushed document to Etherpad" padid="%s" token="%s"' % (docid, acctok[-20:])) + log.info(f'msg="Pushed document to Etherpad" padid="{docid}" token="{acctok[-20:]}"') except requests.exceptions.ConnectionError as e: - log.error('msg="Exception raised attempting to connect to Etherpad" method="import" exception="%s"' % e) + log.error(f'msg="Exception raised attempting to connect to Etherpad" method="import" exception="{e}"') raise AppFailure # generate and return a WOPI lock structure for this document return wopic.generatelock(docid, filemd, epfile, acctok, False) @@ -144,7 +144,7 @@ def _fetchfrometherpad(wopilock, acctok): raise AppFailure return res.content except requests.exceptions.ConnectionError as e: - log.error('msg="Exception raised attempting to connect to Etherpad" exception="%s"' % e) + log.error(f'msg="Exception raised attempting to connect to Etherpad" exception="{e}"') raise AppFailure diff --git a/src/bridge/wopiclient.py b/src/bridge/wopiclient.py index 7f5962b7..a6f59f28 100644 --- a/src/bridge/wopiclient.py +++ b/src/bridge/wopiclient.py @@ -38,12 +38,12 @@ def request(wopisrc, acctok, method, contents=None, headers=None): log.debug('msg="Calling WOPI" url="%s" headers="%s" acctok="%s" ssl="%s"' % (wopiurl, headers, acctok[-20:], sslverify)) if method == 'GET': - return requests.get('%s?access_token=%s' % (wopiurl, acctok), verify=sslverify) + return requests.get(f'{wopiurl}?access_token={acctok}', verify=sslverify) if method == 'POST': - return requests.post('%s?access_token=%s' % (wopiurl, acctok), verify=sslverify, + return requests.post(f'{wopiurl}?access_token={acctok}', verify=sslverify, headers=headers, data=contents) except (requests.exceptions.ConnectionError, IOError) as e: - log.error('msg="Unable to contact WOPI" wopiurl="%s" acctok="%s" response="%s"' % (wopiurl, acctok, e)) + log.error(f'msg="Unable to contact WOPI" wopiurl="{wopiurl}" acctok="{acctok}" response="{e}"') res = Response() res.status_code = http.client.INTERNAL_SERVER_ERROR return res @@ -74,7 +74,7 @@ def checkfornochanges(content, wopilock, acctokforlog): h = hashlib.sha1() h.update(content) if h.hexdigest() == wopilock['dig']: - log.info('msg="File unchanged, skipping save" token="%s"' % acctokforlog[-20:]) + log.info(f'msg="File unchanged, skipping save" token="{acctokforlog[-20:]}"') return True return False @@ -89,7 +89,7 @@ def getlock(wopisrc, acctok): # the lock is expected to be a JSON dict, see generatelock() return json.loads(res.headers['X-WOPI-Lock']) except (ValueError, KeyError, json.decoder.JSONDecodeError) as e: - log.warning('msg="Missing or malformed WOPI lock" exception="%s: %s"' % (type(e), e)) + log.warning(f'msg="Missing or malformed WOPI lock" exception="{type(e)}: {e}"') raise InvalidLock(e) @@ -119,7 +119,7 @@ def refreshlock(wopisrc, acctok, wopilock, digest=None, toclose=None): return newlock if res.status_code == http.client.CONFLICT: # we have a race condition, another thread has updated the lock before us - log.warning('msg="Got conflict in refreshing lock, retrying" url="%s"' % wopisrc) + log.warning(f'msg="Got conflict in refreshing lock, retrying" url="{wopisrc}"') try: currlock = json.loads(res.headers['X-WOPI-Lock']) except json.decoder.JSONDecodeError as e: @@ -152,7 +152,7 @@ def refreshdigestandlock(wopisrc, acctok, wopilock, content): dig = h.hexdigest() try: wopilock = refreshlock(wopisrc, acctok, wopilock, digest=dig) - log.info('msg="Save completed" filename="%s" dig="%s" token="%s"' % (wopilock['fn'], dig, acctok[-20:])) + log.info(f"msg=\"Save completed\" filename=\"{wopilock['fn']}\" dig=\"{dig}\" token=\"{acctok[-20:]}\"") return jsonify('File saved successfully'), http.client.OK except InvalidLock: return jsonify('File saved, but failed to refresh lock'), http.client.INTERNAL_SERVER_ERROR @@ -199,11 +199,11 @@ def handleputfile(wopicall, wopisrc, res): res.headers.get('X-WOPI-LockFailureReason')), http.client.INTERNAL_SERVER_ERROR if res.status_code == http.client.INTERNAL_SERVER_ERROR: # hopefully this is transient and the server has kept a local copy for later recovery - log.error('msg="Calling WOPI %s failed, will retry" url="%s" response="%s"' % (wopicall, wopisrc, res.status_code)) + log.error(f'msg="Calling WOPI {wopicall} failed, will retry" url="{wopisrc}" response="{res.status_code}"') return jsonify('Error saving the file, will try again'), http.client.FAILED_DEPENDENCY if res.status_code != http.client.OK: # any other error is considered also fatal - log.error('msg="Calling WOPI %s failed" url="%s" response="%s"' % (wopicall, wopisrc, res.status_code)) + log.error(f'msg="Calling WOPI {wopicall} failed" url="{wopisrc}" response="{res.status_code}"') return jsonify('Error saving the file, please contact support'), http.client.INTERNAL_SERVER_ERROR return None @@ -235,7 +235,7 @@ def saveas(wopisrc, acctok, wopilock, targetname, content): log.warning('msg="Failed to delete the previous file" token="%s" response="%d"' % (acctok[-20:], res.status_code)) else: - log.info('msg="Previous file unlocked and removed successfully" token="%s"' % acctok[-20:]) + log.info(f'msg="Previous file unlocked and removed successfully" token="{acctok[-20:]}"') - log.info('msg="Final save completed" filename="%s" token="%s"' % (newname, acctok[-20:])) + log.info(f'msg="Final save completed" filename="{newname}" token="{acctok[-20:]}"') return jsonify('File saved successfully'), http.client.OK diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index 0ab9586d..7c6f393a 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -75,7 +75,7 @@ def authenticate_for_test(userid, userpwd): '''Use basic authentication against Reva for testing purposes''' authReq = cs3gw.AuthenticateRequest(type='basic', client_id=userid, client_secret=userpwd) authRes = ctx['cs3gw'].Authenticate(authReq) - log.debug('msg="Authenticated user" userid="%s"' % authRes.user.id) + log.debug(f'msg="Authenticated user" userid="{authRes.user.id}"') if authRes.status.code != cs3code.CODE_OK: raise IOError('Failed to authenticate as user ' + userid + ': ' + authRes.status.message) return authRes.token @@ -91,7 +91,7 @@ def stat(endpoint, fileref, userid, versioninv=1): tend = time.time() if statInfo.status.code == cs3code.CODE_NOT_FOUND: - log.info('msg="File not found" endpoint="%s" fileref="%s" trace="%s"' % (endpoint, fileref, statInfo.status.trace)) + log.info(f'msg="File not found" endpoint="{endpoint}" fileref="{fileref}" trace="{statInfo.status.trace}"') raise IOError(common.ENOENT_MSG) if statInfo.status.code != cs3code.CODE_OK: log.error('msg="Failed stat" endpoint="%s" fileref="%s" trace="%s" reason="%s"' % @@ -145,7 +145,7 @@ def setxattr(endpoint, filepath, userid, key, value, lockmd): log.error('msg="Failed to setxattr" filepath="%s" key="%s" trace="%s" code="%s" reason="%s"' % (filepath, key, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - log.debug('msg="Invoked setxattr" result="%s"' % res) + log.debug(f'msg="Invoked setxattr" result="{res}"') def getxattr(endpoint, filepath, userid, key): @@ -155,7 +155,7 @@ def getxattr(endpoint, filepath, userid, key): statInfo = ctx['cs3gw'].Stat(request=cs3sp.StatRequest(ref=reference), metadata=[('x-access-token', userid)]) tend = time.time() if statInfo.status.code == cs3code.CODE_NOT_FOUND: - log.debug('msg="Invoked stat for getxattr on missing file" filepath="%s"' % filepath) + log.debug(f'msg="Invoked stat for getxattr on missing file" filepath="{filepath}"') return None if statInfo.status.code != cs3code.CODE_OK: log.error('msg="Failed to stat" filepath="%s" userid="%s" trace="%s" key="%s" reason="%s"' % @@ -165,7 +165,7 @@ def getxattr(endpoint, filepath, userid, key): xattrvalue = statInfo.info.arbitrary_metadata.metadata[key] if xattrvalue == '': raise KeyError - log.debug('msg="Invoked stat for getxattr" filepath="%s" elapsedTimems="%.1f"' % (filepath, (tend - tstart) * 1000)) + log.debug(f'msg="Invoked stat for getxattr" filepath="{filepath}" elapsedTimems="{(tend - tstart) * 1000:.1f}"') return xattrvalue except KeyError: log.info('msg="Empty value or key not found in getxattr" filepath="%s" key="%s" trace="%s" metadata="%s"' % @@ -185,7 +185,7 @@ def rmxattr(endpoint, filepath, userid, key, lockmd): log.error('msg="Failed to rmxattr" filepath="%s" trace="%s" key="%s" reason="%s"' % (filepath, key, res.status.trace, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - log.debug('msg="Invoked rmxattr" result="%s"' % res.status) + log.debug(f'msg="Invoked rmxattr" result="{res.status}"') def setlock(endpoint, filepath, userid, appname, value): @@ -203,7 +203,7 @@ def setlock(endpoint, filepath, userid, appname, value): log.error('msg="Failed to setlock" filepath="%s" appname="%s" value="%s" trace="%s" code="%s" reason="%s"' % (filepath, appname, value, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - log.debug('msg="Invoked setlock" filepath="%s" value="%s" result="%s"' % (filepath, value, res.status)) + log.debug(f'msg="Invoked setlock" filepath="{filepath}" value="{value}" result="{res.status}"') def getlock(endpoint, filepath, userid): @@ -212,13 +212,13 @@ def getlock(endpoint, filepath, userid): req = cs3sp.GetLockRequest(ref=reference) res = ctx['cs3gw'].GetLock(request=req, metadata=[('x-access-token', userid)]) if res.status.code == cs3code.CODE_NOT_FOUND: - log.debug('msg="Invoked getlock on unlocked or missing file" filepath="%s"' % filepath) + log.debug(f'msg="Invoked getlock on unlocked or missing file" filepath="{filepath}"') return None if res.status.code != cs3code.CODE_OK: log.error('msg="Failed to getlock" filepath="%s" trace="%s" code="%s" reason="%s"' % (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - log.debug('msg="Invoked getlock" filepath="%s" result="%s"' % (filepath, res.lock)) + log.debug(f'msg="Invoked getlock" filepath="{filepath}" result="{res.lock}"') # rebuild a dict corresponding to the internal JSON structure used by Reva, cf. commoniface.py return { 'lock_id': res.lock.lock_id, @@ -250,7 +250,7 @@ def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): log.warning('msg="Failed to refreshlock" filepath="%s" appname="%s" value="%s" trace="%s" code="%s" reason="%s"' % (filepath, appname, value, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - log.debug('msg="Invoked refreshlock" filepath="%s" value="%s" result="%s"' % (filepath, value, res.status)) + log.debug(f'msg="Invoked refreshlock" filepath="{filepath}" value="{value}" result="{res.status}"') def unlock(endpoint, filepath, userid, appname, value): @@ -263,7 +263,7 @@ def unlock(endpoint, filepath, userid, appname, value): log.error('msg="Failed to unlock" filepath="%s" trace="%s" code="%s" reason="%s"' % (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - log.debug('msg="Invoked unlock" filepath="%s" value="%s" result="%s"' % (filepath, value, res.status)) + log.debug(f'msg="Invoked unlock" filepath="{filepath}" value="{value}" result="{res.status}"') def readfile(endpoint, filepath, userid, lockid): @@ -275,7 +275,7 @@ def readfile(endpoint, filepath, userid, lockid): req = cs3sp.InitiateFileDownloadRequest(ref=reference, lock_id=lockid) res = ctx['cs3gw'].InitiateFileDownload(request=req, metadata=[('x-access-token', userid)]) if res.status.code == cs3code.CODE_NOT_FOUND: - log.info('msg="File not found on read" filepath="%s"' % filepath) + log.info(f'msg="File not found on read" filepath="{filepath}"') yield IOError(common.ENOENT_MSG) elif res.status.code != cs3code.CODE_OK: log.error('msg="Failed to initiateFileDownload on read" filepath="%s" trace="%s" code="%s" reason="%s"' % @@ -294,7 +294,7 @@ def readfile(endpoint, filepath, userid, lockid): } fileget = requests.get(url=protocol.download_endpoint, headers=headers, verify=ctx['ssl_verify']) except requests.exceptions.RequestException as e: - log.error('msg="Exception when downloading file from Reva" reason="%s"' % e) + log.error(f'msg="Exception when downloading file from Reva" reason="{e}"') yield IOError(e) data = fileget.content if fileget.status_code != http.client.OK: @@ -302,7 +302,7 @@ def readfile(endpoint, filepath, userid, lockid): (fileget.status_code, fileget.reason.replace('"', "'"))) yield IOError(fileget.reason) else: - log.info('msg="File open for read" filepath="%s" elapsedTimems="%.1f"' % (filepath, (tend - tstart) * 1000)) + log.info(f'msg="File open for read" filepath="{filepath}" elapsedTimems="{(tend - tstart) * 1000:.1f}"') for i in range(0, len(data), ctx['chunksize']): yield data[i:i + ctx['chunksize']] @@ -348,10 +348,10 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): } putres = requests.put(url=protocol.upload_endpoint, data=content, headers=headers, verify=ctx['ssl_verify']) except requests.exceptions.RequestException as e: - log.error('msg="Exception when uploading file to Reva" reason="%s"' % e) + log.error(f'msg="Exception when uploading file to Reva" reason="{e}"') raise IOError(e) if putres.status_code == http.client.UNAUTHORIZED: - log.warning('msg="Access denied uploading file to Reva" reason="%s"' % putres.reason) + log.warning(f'msg="Access denied uploading file to Reva" reason="{putres.reason}"') raise IOError(common.ACCESS_ERROR) if putres.status_code != http.client.OK: log.error('msg="Error uploading file to Reva" code="%d" reason="%s"' % (putres.status_code, putres.reason)) @@ -377,7 +377,7 @@ def renamefile(endpoint, filepath, newfilepath, userid, lockmd): log.error('msg="Failed to rename file" filepath="%s" trace="%s" code="%s" reason="%s"' % (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - log.debug('msg="Invoked renamefile" result="%s"' % res) + log.debug(f'msg="Invoked renamefile" result="{res}"') def removefile(endpoint, filepath, userid, _force=False): @@ -388,9 +388,9 @@ def removefile(endpoint, filepath, userid, _force=False): res = ctx['cs3gw'].Delete(request=req, metadata=[('x-access-token', userid)]) if res.status.code != cs3code.CODE_OK: if 'path not found' in str(res): - log.info('msg="Invoked removefile on non-existing file" filepath="%s"' % filepath) + log.info(f'msg="Invoked removefile on non-existing file" filepath="{filepath}"') raise IOError(common.ENOENT_MSG) log.error('msg="Failed to remove file" filepath="%s" trace="%s" code="%s" reason="%s"' % (filepath, res.status.trace, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - log.debug('msg="Invoked removefile" result="%s"' % res) + log.debug(f'msg="Invoked removefile" result="{res}"') diff --git a/src/core/localiface.py b/src/core/localiface.py index d5b6bfb7..240acf8f 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -64,7 +64,7 @@ def init(inconfig, inlog): if not S_ISDIR(mode): raise IOError('Not a directory') except IOError as e: - raise IOError('Could not stat storagehomepath folder %s: %s' % (homepath, e)) + raise IOError(f'Could not stat storagehomepath folder {homepath}: {e}') def getuseridfromcreds(_token, _wopiuser): @@ -113,7 +113,7 @@ def _validatelock(filepath, currlock, lockmd, op, log): raise IOError(common.EXCL_ERROR) if appname and currlock['app_name'] != appname \ and currlock['app_name'] != 'wopi' and appname != 'wopi': # TODO deprecated, to be removed after CERNBox rollout - raise IOError(common.EXCL_ERROR + ', file is locked by %s' % currlock['app_name']) + raise IOError(common.EXCL_ERROR + f", file is locked by {currlock['app_name']}") if value != currlock['lock_id']: raise IOError(common.EXCL_ERROR) except IOError as e: @@ -132,7 +132,7 @@ def setxattr(endpoint, filepath, userid, key, value, lockmd): try: os.setxattr(_getfilepath(filepath), 'user.' + key, str(value).encode()) except OSError as e: - log.error('msg="Failed to setxattr" filepath="%s" key="%s" exception="%s"' % (filepath, key, e)) + log.error(f'msg="Failed to setxattr" filepath="{filepath}" key="{key}" exception="{e}"') raise IOError(e) @@ -141,7 +141,7 @@ def getxattr(_endpoint, filepath, _userid, key): try: return os.getxattr(_getfilepath(filepath), 'user.' + key).decode('UTF-8') except OSError as e: - log.warning('msg="Failed to getxattr or missing key" filepath="%s" key="%s" exception="%s"' % (filepath, key, e)) + log.warning(f'msg="Failed to getxattr or missing key" filepath="{filepath}" key="{key}" exception="{e}"') return None @@ -152,24 +152,24 @@ def rmxattr(endpoint, filepath, userid, key, lockmd): try: os.removexattr(_getfilepath(filepath), 'user.' + key) except OSError as e: - log.error('msg="Failed to rmxattr" filepath="%s" key="%s" exception="%s"' % (filepath, key, e)) + log.error(f'msg="Failed to rmxattr" filepath="{filepath}" key="{key}" exception="{e}"') raise IOError(e) def setlock(endpoint, filepath, userid, appname, value): '''Set the lock as an xattr on behalf of the given userid''' - log.debug('msg="Invoked setlock" filepath="%s" value="%s"' % (filepath, value)) + log.debug(f'msg="Invoked setlock" filepath="{filepath}" value="{value}"') with open(_getfilepath(filepath)) as fd: fl = Flock(fd) # ensures atomicity of the following operations try: with fl: if not getlock(endpoint, filepath, userid): - log.debug('msg="setlock: invoking setxattr" filepath="%s" value="%s"' % (filepath, value)) + log.debug(f'msg="setlock: invoking setxattr" filepath="{filepath}" value="{value}"') setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value), None) else: raise IOError(common.EXCL_ERROR) except BlockingIOError as e: - log.error('msg="File already flocked" filepath="%s" exception="%s"' % (filepath, e)) + log.error(f'msg="File already flocked" filepath="{filepath}" exception="{e}"') raise IOError(common.EXCL_ERROR) @@ -179,10 +179,10 @@ def getlock(endpoint, filepath, _userid): if rawl: lock = common.retrieverevalock(rawl) if lock['expiration']['seconds'] > time.time(): - log.debug('msg="Invoked getlock" filepath="%s"' % filepath) + log.debug(f'msg="Invoked getlock" filepath="{filepath}"') return lock # otherwise, the lock had expired: drop it and return None - log.debug('msg="getlock: removed stale lock" filepath="%s"' % filepath) + log.debug(f'msg="getlock: removed stale lock" filepath="{filepath}"') rmxattr(endpoint, filepath, '0:0', common.LOCKKEY, None) return None @@ -195,36 +195,36 @@ def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): oldvalue = currlock['lock_id'] _validatelock(filepath, currlock, (appname, oldvalue), 'refreshlock', log) # this is non-atomic, but if we get here the lock was already held - log.debug('msg="Invoked refreshlock" filepath="%s" value="%s"' % (filepath, value)) + log.debug(f'msg="Invoked refreshlock" filepath="{filepath}" value="{value}"') setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value), None) def unlock(endpoint, filepath, userid, appname, value): '''Remove the lock as an xattr on behalf of the given userid''' _validatelock(filepath, getlock(endpoint, filepath, userid), (appname, value), 'unlock', log) - log.debug('msg="Invoked unlock" filepath="%s" value="%s"' % (filepath, value)) + log.debug(f'msg="Invoked unlock" filepath="{filepath}" value="{value}"') rmxattr(endpoint, filepath, '0:0', common.LOCKKEY, None) def readfile(_endpoint, filepath, _userid, _lockid): '''Read a file on behalf of the given userid. Note that the function is a generator, managed by the app server.''' - log.debug('msg="Invoking readFile" filepath="%s"' % filepath) + log.debug(f'msg="Invoking readFile" filepath="{filepath}"') try: tstart = time.time() chunksize = config.getint('io', 'chunksize') with open(_getfilepath(filepath), mode='rb', buffering=chunksize) as f: tend = time.time() - log.info('msg="File open for read" filepath="%s" elapsedTimems="%.1f"' % (filepath, (tend - tstart) * 1000)) + log.info(f'msg="File open for read" filepath="{filepath}" elapsedTimems="{(tend - tstart) * 1000:.1f}"') # the actual read is buffered and managed by the app server for chunk in iter(lambda: f.read(chunksize), b''): yield chunk except FileNotFoundError: # log this case as info to keep the logs cleaner - log.info('msg="File not found on read" filepath="%s"' % filepath) + log.info(f'msg="File not found on read" filepath="{filepath}"') # as this is a generator, we yield the error string instead of the file's contents yield IOError('No such file or directory') except OSError as e: - log.error('msg="Error opening the file for read" filepath="%s" error="%s"' % (filepath, e)) + log.error(f'msg="Error opening the file for read" filepath="{filepath}" error="{e}"') yield IOError(e) @@ -254,10 +254,10 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): os.close(fd) # f.close() raises EBADF! while this works # as f goes out of scope here, we'd get a false ResourceWarning, which is ignored by the above filter except FileExistsError: - log.info('msg="File exists on write but islock flag requested" filepath="%s"' % filepath) + log.info(f'msg="File exists on write but islock flag requested" filepath="{filepath}"') raise IOError(common.EXCL_ERROR) except OSError as e: - log.warning('msg="Error writing file in O_EXCL mode" filepath="%s" error="%s"' % (filepath, e)) + log.warning(f'msg="Error writing file in O_EXCL mode" filepath="{filepath}" error="{e}"') raise IOError(e) else: try: @@ -265,7 +265,7 @@ def writefile(endpoint, filepath, userid, content, lockmd, islock=False): tend = time.time() written = f.write(content) except OSError as e: - log.error('msg="Error writing file" filepath="%s" error="%s"' % (filepath, e)) + log.error(f'msg="Error writing file" filepath="{filepath}" error="{e}"') raise IOError(e) if written != size: raise IOError('Written %d bytes but content is %d bytes' % (written, size)) diff --git a/src/core/wopi.py b/src/core/wopi.py index 37a7b590..8bccf93b 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -45,12 +45,12 @@ def checkFileInfo(fileid, acctok): fmd['PostMessageOrigin'] = host.scheme + '://' + host.netloc fmd['EditModePostMessage'] = fmd['EditNotificationPostMessage'] = True else: - fmd['HostEditUrl'] = '%s%s%s' % (acctok['appediturl'], '&' if '?' in acctok['appediturl'] else '?', wopiSrc) + fmd['HostEditUrl'] = f"{acctok['appediturl']}{'&' if '?' in acctok['appediturl'] else '?'}{wopiSrc}" hostvurl = srv.config.get('general', 'hostviewurl', fallback=None) if hostvurl: fmd['HostViewUrl'] = utils.generateUrlFromTemplate(hostvurl, acctok) else: - fmd['HostViewUrl'] = '%s%s%s' % (acctok['appviewurl'], '&' if '?' in acctok['appviewurl'] else '?', wopiSrc) + fmd['HostViewUrl'] = f"{acctok['appviewurl']}{'&' if '?' in acctok['appviewurl'] else '?'}{wopiSrc}" fsurl = srv.config.get('general', 'filesharingurl', fallback=None) if fsurl: fmd['FileSharingUrl'] = utils.generateUrlFromTemplate(fsurl, acctok) @@ -79,7 +79,7 @@ def checkFileInfo(fileid, acctok): fmd['UserId'] = acctok['wopiuser'].split('!')[-1] # typically same as OwnerId; different when accessing shared documents fmd['Size'] = statInfo['size'] # note that in ownCloud 10 the version is generated as: `'V' + etag + checksum` - fmd['Version'] = 'v%s' % statInfo['etag'] + fmd['Version'] = f"v{statInfo['etag']}" fmd['SupportsExtendedLockLength'] = fmd['SupportsGetLock'] = True fmd['SupportsUpdate'] = fmd['UserCanWrite'] = fmd['SupportsLocks'] = \ fmd['SupportsDeleteFile'] = acctok['viewmode'] == utils.ViewMode.READ_WRITE @@ -113,7 +113,7 @@ def checkFileInfo(fileid, acctok): # redact sensitive metadata for the logs fmd['HostViewUrl'] = fmd['HostEditUrl'] = fmd['DownloadUrl'] = fmd['FileUrl'] = \ fmd['BreadcrumbBrandUrl'] = fmd['FileSharingUrl'] = '_redacted_' - log.info('msg="File metadata response" token="%s" metadata="%s"' % (flask.request.args['access_token'][-20:], fmd)) + log.info(f"msg=\"File metadata response\" token=\"{flask.request.args['access_token'][-20:]}\" metadata=\"{fmd}\"") return res except IOError as e: log.info('msg="Requested file not found" filename="%s" token="%s" details="%s"' % @@ -138,10 +138,10 @@ def getFile(_fileid, acctok): # stream file from storage to client resp = flask.Response(f, mimetype='application/octet-stream') resp.status_code = http.client.OK - resp.headers['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename(acctok['filename']) + resp.headers['Content-Disposition'] = f"attachment; filename=\"{os.path.basename(acctok['filename'])}\"" resp.headers['X-Frame-Options'] = 'sameorigin' resp.headers['X-XSS-Protection'] = '1; mode=block' - resp.headers['X-WOPI-ItemVersion'] = 'v%s' % statInfo['etag'] + resp.headers['X-WOPI-ItemVersion'] = f"v{statInfo['etag']}" return resp except StopIteration: # File is empty, still return OK (strictly speaking, we should return 204 NO_CONTENT) @@ -253,7 +253,7 @@ def setLock(fileid, reqheaders, acctok): log.warning('msg="Unable to set lastwritetime xattr" lockop="%s" user="%s" filename="%s" token="%s" reason="%s"' % (op.title(), acctok['userid'][-20:], fn, flask.request.args['access_token'][-20:], e)) - return utils.makeLockSuccessResponse(op, acctok, lock, oldLock, 'v%s' % statInfo['etag']) + return utils.makeLockSuccessResponse(op, acctok, lock, oldLock, f"v{statInfo['etag']}") except IOError as e: if common.EXCL_ERROR in str(e): @@ -273,7 +273,7 @@ def setLock(fileid, reqheaders, acctok): evicted = utils.checkAndEvictLock(acctok['userid'], acctok['appname'], retrievedLock, oldLock, lock, acctok['endpoint'], fn, int(statInfo['mtime'])) if evicted: - return utils.makeLockSuccessResponse(op, acctok, lock, oldLock, 'v%s' % statInfo['etag']) + return utils.makeLockSuccessResponse(op, acctok, lock, oldLock, f"v{statInfo['etag']}") else: return utils.makeConflictResponse(op, acctok['userid'], retrievedLock, lock, oldLock, fn, 'The file is locked by %s' % @@ -284,7 +284,7 @@ def setLock(fileid, reqheaders, acctok): try: st.refreshlock(acctok['endpoint'], fn, acctok['userid'], acctok['appname'], utils.encodeLock(lock), utils.encodeLock(oldLock)) - return utils.makeLockSuccessResponse(op, acctok, lock, oldLock, 'v%s' % statInfo['etag']) + return utils.makeLockSuccessResponse(op, acctok, lock, oldLock, f"v{statInfo['etag']}") except IOError as rle: # this is unexpected now log.error('msg="Failed to refresh lock" lockop="%s" filename="%s" token="%s" lock="%s" error="%s"' % @@ -347,7 +347,7 @@ def unlock(fileid, reqheaders, acctok): pass resp = flask.Response() resp.status_code = http.client.OK - resp.headers['X-WOPI-ItemVersion'] = 'v%s' % statInfo['etag'] + resp.headers['X-WOPI-ItemVersion'] = f"v{statInfo['etag']}" return resp @@ -458,7 +458,7 @@ def putRelative(fileid, reqheaders, acctok): 'endpoint': acctok['endpoint'], 'fileid': newfileid, } - newwopisrc = '%s&access_token=%s' % (utils.generateWopiSrc(inode, acctok['appname'] == srv.proxiedappname), newacctok) + newwopisrc = f"{utils.generateWopiSrc(inode, acctok['appname'] == srv.proxiedappname)}&access_token={newacctok}" putrelmd = { 'Name': os.path.basename(targetName), 'Url': url_unquote(newwopisrc).replace('&access_token', '?access_token'), @@ -467,15 +467,15 @@ def putRelative(fileid, reqheaders, acctok): if hosteurl: putrelmd['HostEditUrl'] = utils.generateUrlFromTemplate(hosteurl, mdforhosturls) else: - putrelmd['HostEditUrl'] = '%s%s%s' % (acctok['appediturl'], '&' if '?' in acctok['appediturl'] else '?', newwopisrc) + putrelmd['HostEditUrl'] = f"{acctok['appediturl']}{'&' if '?' in acctok['appediturl'] else '?'}{newwopisrc}" hostvurl = srv.config.get('general', 'hostviewurl', fallback=None) if hostvurl: putrelmd['HostViewUrl'] = utils.generateUrlFromTemplate(hostvurl, mdforhosturls) else: - putrelmd['HostViewUrl'] = '%s%s%s' % (acctok['appviewurl'], '&' if '?' in acctok['appviewurl'] else '?', newwopisrc) + putrelmd['HostViewUrl'] = f"{acctok['appviewurl']}{'&' if '?' in acctok['appviewurl'] else '?'}{newwopisrc}" resp = flask.Response(json.dumps(putrelmd), mimetype='application/json') putrelmd['Url'] = putrelmd['HostEditUrl'] = putrelmd['HostViewUrl'] = '_redacted_' - log.info('msg="PutRelative response" token="%s" metadata="%s"' % (newacctok[-20:], putrelmd)) + log.info(f'msg="PutRelative response" token="{newacctok[-20:]}" metadata="{putrelmd}"') return resp @@ -492,7 +492,7 @@ def deleteFile(fileid, _reqheaders_unused, acctok): except IOError as e: if common.ENOENT_MSG in str(e): return 'File not found', http.client.NOT_FOUND - log.error('msg="DeleteFile" token="%s" error="%s"' % (flask.request.args['access_token'][-20:], e)) + log.error(f"msg=\"DeleteFile\" token=\"{flask.request.args['access_token'][-20:]}\" error=\"{e}\"") return IO_ERROR, http.client.INTERNAL_SERVER_ERROR @@ -530,7 +530,7 @@ def renameFile(fileid, reqheaders, acctok): # send the response as JSON return flask.Response(json.dumps(renamemd), mimetype='application/json') except IOError as e: - log.warn('msg="RenameFile" token="%s" error="%s"' % (flask.request.args['access_token'][-20:], e)) + log.warn(f"msg=\"RenameFile\" token=\"{flask.request.args['access_token'][-20:]}\" error=\"{e}\"") resp = flask.Response() if common.ENOENT_MSG in str(e): resp.headers['X-WOPI-InvalidFileNameError'] = 'File not found' @@ -539,7 +539,7 @@ def renameFile(fileid, reqheaders, acctok): resp.headers['X-WOPI-InvalidFileNameError'] = 'Cannot rename/move unlocked file' resp.status_code = http.client.NOT_IMPLEMENTED else: - resp.headers['X-WOPI-InvalidFileNameError'] = 'Failed to rename: %s' % e + resp.headers['X-WOPI-InvalidFileNameError'] = f'Failed to rename: {e}' resp.status_code = http.client.INTERNAL_SERVER_ERROR return resp @@ -604,7 +604,7 @@ def putFile(fileid, acctok): (acctok['userid'][-20:], acctok['filename'], statInfo['etag'], flask.request.args['access_token'][-20:])) resp = flask.Response() resp.status_code = http.client.OK - resp.headers['X-WOPI-ItemVersion'] = 'v%s' % statInfo['etag'] + resp.headers['X-WOPI-ItemVersion'] = f"v{statInfo['etag']}" return resp except IOError as e: diff --git a/src/core/wopiutils.py b/src/core/wopiutils.py index efd3c38b..bc8ddc82 100644 --- a/src/core/wopiutils.py +++ b/src/core/wopiutils.py @@ -92,7 +92,7 @@ def facade(*args, **kwargs): m = f[f.rfind('/') + 1:] try: # as we use a `key="value" ...` format in all logs, we only have args[0] - payload = 'module="%s" %s ' % (m, args[0]) + payload = f'module="{m}" {args[0]} ' # now convert the payload to a dictionary assuming no `="` nor `" ` is present inside any key or value! # the added trailing space matches the `" ` split, so we remove the last element of that list payload = dict([tuple(kv.split('="')) for kv in payload.split('" ')[:-1]]) @@ -100,7 +100,7 @@ def facade(*args, **kwargs): payload = str(json.dumps(payload))[1:-1] except Exception: # pylint: disable=broad-except # if the above assumptions do not hold, just json-escape the original log - payload = '"module": "%s", "payload": "%s"' % (m, json.dumps(args[0])) + payload = f'"module": "{m}", "payload": "{json.dumps(args[0])}"' args = (payload,) # pass-through facade return getattr(self.logger, name)(*args, **kwargs) @@ -167,14 +167,14 @@ def validateAndLogHeaders(op): def generateWopiSrc(fileid, proxy=False): '''Returns a URL-encoded WOPISrc for the given fileid, proxied if required.''' if not proxy or not srv.wopiproxy: - return url_quote_plus('%s/wopi/files/%s' % (srv.wopiurl, fileid)).replace('-', '%2D') + return url_quote_plus(f'{srv.wopiurl}/wopi/files/{fileid}').replace('-', '%2D') # proxy the WOPI request through an external WOPI proxy service, but only if it was not already proxied if len(fileid) < 90: # heuristically, proxied fileids are (much) longer than that - log.debug('msg="Generating proxied fileid" fileid="%s" proxy="%s"' % (fileid, srv.wopiproxy)) + log.debug(f'msg="Generating proxied fileid" fileid="{fileid}" proxy="{srv.wopiproxy}"') fileid = jwt.encode({'u': srv.wopiurl + '/wopi/files/', 'f': fileid}, srv.wopiproxykey, algorithm='HS256') else: - log.debug('msg="Proxied fileid already created" fileid="%s" proxy="%s"' % (fileid, srv.wopiproxy)) - return url_quote_plus('%s/wopi/files/%s' % (srv.wopiproxy, fileid)).replace('-', '%2D') + log.debug(f'msg="Proxied fileid already created" fileid="{fileid}" proxy="{srv.wopiproxy}"') + return url_quote_plus(f'{srv.wopiproxy}/wopi/files/{fileid}').replace('-', '%2D') def generateUrlFromTemplate(url, acctok): @@ -219,7 +219,7 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app # the inode serves as fileid (and must not change across save operations) statinfo = st.statx(endpoint, fileid, userid) except IOError as e: - log.info('msg="Requested file not found or not a file" fileid="%s" error="%s"' % (fileid, e)) + log.info(f'msg="Requested file not found or not a file" fileid="{fileid}" error="{e}"') raise exptime = int(time.time()) + srv.tokenvalidity fext = os.path.splitext(statinfo['filepath'])[1].lower() @@ -239,13 +239,13 @@ def generateAccessToken(userid, fileid, viewmode, user, folderurl, endpoint, app fext[1:3] in ('od', 'ot') and appname not in ('Collabora', '') and viewmode == ViewMode.READ_WRITE: # we're opening an ODF (`.o[d|t]?`) file and the app is not Collabora (the appname may be empty because the legacy # endpoint does not set appname when the app is not proxied, so we optimistically assume it's Collabora and let it go) - log.info('msg="Forcing read-only access to ODF file" filename="%s"' % statinfo['filepath']) + log.info(f"msg=\"Forcing read-only access to ODF file\" filename=\"{statinfo['filepath']}\"") viewmode = ViewMode.READ_ONLY tokmd = { 'userid': userid, 'wopiuser': wopiuser, 'usertype': usertype.value, 'filename': statinfo['filepath'], 'fileid': fileid, 'username': friendlyname, 'viewmode': viewmode.value, 'folderurl': folderurl, 'endpoint': endpoint, 'appname': appname, 'appediturl': appediturl, 'appviewurl': appviewurl, - 'exp': exptime, 'iss': 'cs3org:wopiserver:%s' % WOPIVER # standard claims + 'exp': exptime, 'iss': f'cs3org:wopiserver:{WOPIVER}' # standard claims } if forcelock: tokmd['forcelock'] = '1' @@ -349,10 +349,10 @@ def compareWopiLocks(lock1, lock2): a bug in Word Online, currently the internal format of the WOPI locks is looked at, based on heuristics. Note that this format is subject to change and is not documented!''' if lock1 == lock2: - log.debug('msg="compareLocks" lock1="%s" lock2="%s" result="True"' % (lock1, lock2)) + log.debug(f'msg="compareLocks" lock1="{lock1}" lock2="{lock2}" result="True"') return True if srv.config.get('general', 'wopilockstrictcheck', fallback='False').upper() == 'TRUE': - log.debug('msg="compareLocks" lock1="%s" lock2="%s" strict="True" result="False"' % (lock1, lock2)) + log.debug(f'msg="compareLocks" lock1="{lock1}" lock2="{lock2}" strict="True" result="False"') return False # before giving up, attempt to parse the lock as a JSON dictionary if allowed by the config @@ -373,7 +373,7 @@ def compareWopiLocks(lock1, lock2): except (TypeError, ValueError): # lock1 is not a JSON dictionary: log the lock values and fail the comparison pass - log.debug('msg="compareLocks" lock1="%s" lock2="%s" strict="False" result="False"' % (lock1, lock2)) + log.debug(f'msg="compareLocks" lock1="{lock1}" lock2="{lock2}" strict="False" result="False"') return False @@ -442,7 +442,7 @@ def makeConflictResponse(operation, user, retrievedlock, lock, oldlock, filename 'type': os.path.splitext(filename)[1], } if savetime: - fileage = '%1.1f' % (time.time() - int(savetime)) + fileage = f'{time.time() - int(savetime):1.1f}' else: fileage = 'NA' log.warning('msg="Returning conflict" lockop="%s" user="%s" filename="%s" token="%s" sessionId="%s" lock="%s" ' @@ -519,7 +519,7 @@ def storeAfterConflict(acctok, retrievedlock, lock, reason): next to the original one, or to the user's home, or to the recovery path.''' newname, ext = os.path.splitext(acctok['filename']) # typical EFSS formats are like '_conflict--