feat(backup): support export and import of backups on Web, iOS and Android [WPB-10575]

backup/build.gradle.kts

@@ -56,8 +56,12 @@ kotlin {
         generateTypeScriptDefinitions()
     }
     sourceSets {
+        val androidUnitTest by getting { }
+        remove(androidUnitTest) // Android Unit tests are removed, as we run Instrumentation tests for encryption/file manipulation, etc.
         all {
             languageSettings.optIn("kotlin.ExperimentalUnsignedTypes")
+            languageSettings.optIn("kotlin.experimental.ExperimentalObjCRefinement")
+            languageSettings.optIn("kotlin.js.ExperimentalJsExport")
         }
         val commonMain by getting {
             dependencies {
@@ -132,5 +136,10 @@ kotlin {
                 implementation(libs.pbandk.runtime.macArm64)
             }
         }
+        val jsMain by getting {
+            dependencies {
+                implementation(npm("jszip", "3.10.1"))
+            }
+        }
     }
 }

backup/src/commonMain/kotlin/com/wire/backup/compression/Zipper.kt

@@ -0,0 +1,26 @@
+/*
+ * Wire
+ * Copyright (C) 2024 Wire Swiss GmbH
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/.
+ */
+package com.wire.backup.compression
+
+import com.wire.backup.filesystem.BackupEntry
+import kotlinx.coroutines.Deferred
+import okio.Source
+
+internal interface Zipper {
+    fun archive(data: List<BackupEntry>): Deferred<Source>
+}

backup/src/commonMain/kotlin/com/wire/backup/dump/MPBackupExporter.kt

@@ -17,79 +17,169 @@
  */
 package com.wire.backup.dump
 
+import com.wire.backup.compression.Zipper
 import com.wire.backup.data.BackupConversation
 import com.wire.backup.data.BackupMessage
 import com.wire.backup.data.BackupQualifiedId
 import com.wire.backup.data.BackupUser
 import com.wire.backup.data.toProtoModel
+import com.wire.backup.encryption.EncryptedStream
+import com.wire.backup.encryption.XChaChaPoly1305AuthenticationData
+import com.wire.backup.envelope.BackupHeader
+import com.wire.backup.envelope.BackupHeaderSerializer
+import com.wire.backup.envelope.HashData
+import com.wire.backup.filesystem.BackupEntry
+import com.wire.backup.filesystem.EntryStorage
 import com.wire.backup.ingest.MPBackupMapper
 import com.wire.kalium.protobuf.backup.BackupData
 import com.wire.kalium.protobuf.backup.BackupInfo
+import com.wire.kalium.protobuf.backup.ExportUser
+import com.wire.kalium.protobuf.backup.ExportedConversation
+import com.wire.kalium.protobuf.backup.ExportedMessage
 import kotlinx.datetime.Clock
+import okio.Buffer
+import okio.Sink
+import okio.Source
+import okio.buffer
+import okio.use
 import pbandk.encodeToByteArray
-import kotlin.experimental.ExperimentalObjCName
-import kotlin.experimental.ExperimentalObjCRefinement
 import kotlin.js.JsExport
-import kotlin.native.ObjCName
-import kotlin.native.ShouldRefineInSwift
+import kotlin.js.JsName
 
 /**
  * Entity able to serialize [BackupData] entities, like [BackupMessage], [BackupConversation], [BackupUser]
  * into a cross-platform [BackupData] format.
  */
-@OptIn(ExperimentalObjCName::class, ExperimentalObjCRefinement::class)
 @JsExport
 public abstract class CommonMPBackupExporter(
     private val selfUserId: BackupQualifiedId
 ) {
     private val mapper = MPBackupMapper()
-    private val allUsers = mutableListOf<BackupUser>()
-    private val allConversations = mutableListOf<BackupConversation>()
-    private val allMessages = mutableListOf<BackupMessage>()
-
-    // TODO: Replace `ObjCName` with `JsName` in the future and flip it around.
-    //       Unfortunately the IDE doesn't understand this right now and
-    //       keeps complaining if making the other way around
-    @ObjCName("add")
-    public fun addUser(user: BackupUser) {
-        allUsers.add(user)
+    private val usersChunk = mutableListOf<ExportUser>()
+    private val conversationsChunk = mutableListOf<ExportedConversation>()
+    private val messagesChunk = mutableListOf<ExportedMessage>()
+    private var persistedUserChunks = 0
+    private var persistedConversationsChunks = 0
+    private var persistedMessagesChunks = 0
+
+    private val backupInfo by lazy {
+        BackupInfo(
+            platform = "Common",
+            version = "1.0",
+            userId = selfUserId.toProtoModel(),
+            creationTime = Clock.System.now().toEpochMilliseconds(),
+            clientId = "lol"
+        )
+    }
+
+    @JsName("addUser")
+    public fun add(user: BackupUser) {
+        usersChunk.add(mapper.mapUserToProtobuf(user))
+        if (usersChunk.size > ITEMS_CHUNK_SIZE) {
+            flushUsers()
+        }
     }
 
-    @ObjCName("add")
-    public fun addConversation(conversation: BackupConversation) {
-        allConversations.add(conversation)
+    private fun flushUsers() {
+        if (usersChunk.isEmpty()) return
+        val backupData = BackupData(backupInfo, users = usersChunk)
+        storage.persistEntry(BackupEntry(USERS_ENTRY_PREFIX + persistedUserChunks + ENTRY_SUFFIX, backupData.asSource()))
+        persistedUserChunks++
+        usersChunk.clear()
     }
 
-    @ObjCName("add")
-    public fun addMessage(message: BackupMessage) {
-        allMessages.add(message)
+    @JsName("addConversation")
+    public fun add(conversation: BackupConversation) {
+        conversationsChunk.add(mapper.mapConversationToProtobuf(conversation))
+        if (conversationsChunk.size > ITEMS_CHUNK_SIZE) {
+            flushConversations()
+        }
+    }
+
+    private fun flushConversations() {
+        if (conversationsChunk.isEmpty()) return
+        val backupData = BackupData(backupInfo, conversations = conversationsChunk)
+        storage.persistEntry(BackupEntry(CONVERSATIONS_ENTRY_PREFIX + persistedConversationsChunks + ENTRY_SUFFIX, backupData.asSource()))
+        persistedConversationsChunks++
+        conversationsChunk.clear()
+    }
+
+    @JsName("addMessage")
+    public fun add(message: BackupMessage) {
+        messagesChunk.add(mapper.mapMessageToProtobuf(message))
+        if (messagesChunk.size > ITEMS_CHUNK_SIZE) {
+            flushMessages()
+        }
     }
 
-    @OptIn(ExperimentalStdlibApi::class)
-    @ShouldRefineInSwift // Hidden in Swift
-    public fun serialize(): ByteArray {
-        val backupData = BackupData(
-            BackupInfo(
-                platform = "Common",
-                version = "1.0",
-                userId = selfUserId.toProtoModel(),
-                creationTime = Clock.System.now().toEpochMilliseconds(),
-                clientId = "lol"
-            ),
-            allConversations.map {
-                mapper.mapConversationToProtobuf(it)
-            },
-            allMessages.map {
-                mapper.mapMessageToProtobuf(it)
-            },
-            allUsers.map {
-                mapper.mapUserToProtobuf(it)
-            },
+    private fun flushMessages() {
+        if (messagesChunk.isEmpty()) return
+        val backupData = BackupData(backupInfo, messages = messagesChunk)
+        storage.persistEntry(BackupEntry(MESSAGES_ENTRY_PREFIX + persistedMessagesChunks + ENTRY_SUFFIX, backupData.asSource()))
+        persistedMessagesChunks++
+        messagesChunk.clear()
+    }
+
+    private fun flushAll() {
+        flushUsers()
+        flushConversations()
+        flushMessages()
+    }
+
+    private fun BackupData.asSource(): Source {
+        val buffer = Buffer()
+        return buffer.write(this.encodeToByteArray())
+    }
+
+    internal suspend fun finalize(password: String?, output: Sink) {
+        flushAll()
+        val zippedData = zipper.archive(storage.listEntries()).await()
+        val salt = XChaChaPoly1305AuthenticationData.newSalt()
+
+        val header = BackupHeader(
+            BackupHeaderSerializer.Default.CURRENT_HEADER_VERSION,
+            false,
+            HashData.defaultFromUserId(selfUserId)
         )
-        return backupData.encodeToByteArray().also {
-            println("XPlatform Backup POC. Exported data bytes: ${it.toHexString()}")
+        val headerBytes = BackupHeaderSerializer.Default.headerToBytes(header)
+        output.buffer().use { bufferedOutput ->
+            bufferedOutput.write(headerBytes)
+            bufferedOutput.flush()
+            if (password.isNullOrBlank()) {
+                // We should skip the encryption headers, leaving empty/zeroed bytes
+                val skip = ByteArray(EncryptedStream.XCHACHA_20_POLY_1305_HEADER_LENGTH) { 0x00 }
+                bufferedOutput.write(skip)
+                bufferedOutput.writeAll(zippedData)
+            } else {
+                EncryptedStream.encrypt(
+                    zippedData,
+                    bufferedOutput,
+                    XChaChaPoly1305AuthenticationData(
+                        password,
+                        salt,
+                        headerBytes.toUByteArray(),
+                        header.hashData.operationsLimit,
+                        header.hashData.hashingMemoryLimit
+                    )
+                )
+            }
+            bufferedOutput
         }
     }
+
+    internal abstract val storage: EntryStorage
+    internal abstract val zipper: Zipper
+
+    private companion object {
+        /**
+         * Amount of items (conversations or messages) to be put into a single page / entry
+         */
+        const val ITEMS_CHUNK_SIZE = 1_000
+        const val USERS_ENTRY_PREFIX = "users"
+        const val CONVERSATIONS_ENTRY_PREFIX = "conversations"
+        const val MESSAGES_ENTRY_PREFIX = "messages"
+        const val ENTRY_SUFFIX = ".binpb"
+    }
 }
 
 public expect class MPBackupExporter : CommonMPBackupExporter

backup/src/commonMain/kotlin/com/wire/backup/encryption/EncryptedStream.kt

@@ -17,7 +17,6 @@
  */
 package com.wire.backup.encryption
 
-import com.ionspin.kotlin.crypto.LibsodiumInitializer
 import com.ionspin.kotlin.crypto.pwhash.PasswordHash
 import com.ionspin.kotlin.crypto.pwhash.crypto_pwhash_SALTBYTES
 import com.ionspin.kotlin.crypto.pwhash.crypto_pwhash_argon2id_ALG_ARGON2ID13
@@ -27,7 +26,8 @@ import com.ionspin.kotlin.crypto.secretstream.crypto_secretstream_xchacha20poly1
 import com.ionspin.kotlin.crypto.secretstream.crypto_secretstream_xchacha20poly1305_TAG_FINAL
 import com.ionspin.kotlin.crypto.secretstream.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE
 import com.ionspin.kotlin.crypto.stream.crypto_stream_chacha20_KEYBYTES
-import com.wire.backup.envelope.cryptography.BackupPassphrase
+import com.wire.backup.hash.HASH_MEM_LIMIT
+import com.wire.backup.hash.HASH_OPS_LIMIT
 import okio.Buffer
 import okio.Sink
 import okio.Source
@@ -48,19 +48,17 @@ internal interface EncryptedStream<AuthenticationData> {
      * @return a [UByteArray] containing the extra data that might be needed for decryption depending on the implementation.
      * @see decrypt
      */
-    suspend fun encrypt(source: Source, outputSink: Sink, authenticationData: AuthenticationData): UByteArray
+    suspend fun encrypt(source: Source, outputSink: Sink, authenticationData: AuthenticationData)
 
     /**
-     * Decrypts the [source] data using the provided [authenticationData] and [encryptionHeader].
-     * @param encryptionHeader the result of the [encrypt] function.
+     * Decrypts the [source] data using the provided [authenticationData].
      * The result is fed into the [outputSink].
      * @see encrypt
      */
     suspend fun decrypt(
         source: Source,
         outputSink: Sink,
         authenticationData: AuthenticationData,
-        encryptionHeader: UByteArray
     ): DecryptionResult
 
     /**
@@ -69,24 +67,20 @@ internal interface EncryptedStream<AuthenticationData> {
      * It will encrypt the whole [Source] into an output [Sink] in smaller messages of [INDIVIDUAL_PLAINTEXT_MESSAGE_SIZE].
      */
     companion object XChaCha20Poly1305 : EncryptedStream<XChaChaPoly1305AuthenticationData> {
+        const val XCHACHA_20_POLY_1305_HEADER_LENGTH = 24
         private const val KEY_LENGTH = crypto_stream_chacha20_KEYBYTES
         private const val INDIVIDUAL_PLAINTEXT_MESSAGE_SIZE = 4096L
         private val INDIVIDUAL_ENCRYPTED_MESSAGE_SIZE = INDIVIDUAL_PLAINTEXT_MESSAGE_SIZE + crypto_secretstream_xchacha20poly1305_ABYTES
 
-        private suspend fun initializeLibSodiumIfNeeded() {
-            if (!LibsodiumInitializer.isInitialized()) {
-                LibsodiumInitializer.initialize()
-            }
-        }
-
-        override suspend fun encrypt(source: Source, outputSink: Sink, authenticationData: XChaChaPoly1305AuthenticationData): UByteArray {
+        override suspend fun encrypt(source: Source, outputSink: Sink, authenticationData: XChaChaPoly1305AuthenticationData) {
             initializeLibSodiumIfNeeded()
             val key = generateChaCha20Key(authenticationData)
             val stateHeader = SecretStream.xChaCha20Poly1305InitPush(key)
             val state = stateHeader.state
             val chaChaHeader = stateHeader.header
             val readBuffer = Buffer()
             val output = outputSink.buffer()
+            output.write(chaChaHeader.toByteArray())
 
             // iterate with stuff
             var readBytes: Long
@@ -114,23 +108,22 @@ internal interface EncryptedStream<AuthenticationData> {
             source.close()
             output.close()
             outputSink.close()
-            return chaChaHeader
         }
 
         override suspend fun decrypt(
             source: Source,
             outputSink: Sink,
             authenticationData: XChaChaPoly1305AuthenticationData,
-            encryptionHeader: UByteArray
         ): DecryptionResult {
             initializeLibSodiumIfNeeded()
             var decryptedDataSize = 0L
             val outputBuffer = outputSink.buffer()
             val readBuffer = Buffer()
             val key = generateChaCha20Key(authenticationData)
             return try {
-
-                val stateHeader = SecretStream.xChaCha20Poly1305InitPull(key, encryptionHeader)
+                val headerBuffer = Buffer()
+                source.read(headerBuffer, XCHACHA_20_POLY_1305_HEADER_LENGTH.toLong())
+                val stateHeader = SecretStream.xChaCha20Poly1305InitPull(key, headerBuffer.readByteArray().toUByteArray())
                 var hasReadLastMessage = false
                 while (!hasReadLastMessage) {
                     val readBytes = source.read(readBuffer, INDIVIDUAL_ENCRYPTED_MESSAGE_SIZE)
@@ -157,7 +150,7 @@ internal interface EncryptedStream<AuthenticationData> {
 
         private fun generateChaCha20Key(authData: XChaChaPoly1305AuthenticationData): UByteArray = PasswordHash.pwhash(
             outputLength = KEY_LENGTH,
-            password = authData.passphrase.value,
+            password = authData.passphrase,
             salt = authData.salt,
             opsLimit = authData.hashOpsLimit,
             memLimit = authData.hashMemLimit,
@@ -190,20 +183,14 @@ internal sealed interface DecryptionResult {
  * will also fail. The idea is to do not trust any fruit from the poisoned tree.
  */
 internal data class XChaChaPoly1305AuthenticationData(
-    val passphrase: BackupPassphrase,
+    val passphrase: String,
     val salt: UByteArray,
     val additionalData: UByteArray = ubyteArrayOf(),
     val hashOpsLimit: ULong = HASH_OPS_LIMIT,
     val hashMemLimit: Int = HASH_MEM_LIMIT,
 ) {
     companion object {
-        const val SALT_LENGTH = crypto_pwhash_SALTBYTES
-
-        // crypto_pwhash_argon2i_OPSLIMIT_INTERACTIVE
-        private const val HASH_OPS_LIMIT = 4UL
-
-        // crypto_pwhash_argon2i_MEMLIMIT_INTERACTIVE
-        private const val HASH_MEM_LIMIT = 33554432
-        fun newSalt() = Random.nextUBytes(SALT_LENGTH)
+        private const val SALT_LENGTH = crypto_pwhash_SALTBYTES
+        fun newSalt() = Random.nextUBytes(SALT_LENGTH).toUByteArray()
     }
 }