Implement OneDrive Business Shared Folders Support (Issue #459) (#473)

* Implement OneDrive Business Shared Folders Support

Makefile.in

@@ -54,7 +54,7 @@ endif
 system_unit_files = contrib/systemd/[email protected]
 user_unit_files = contrib/systemd/onedrive.service
 
-DOCFILES = README.md config LICENSE CHANGELOG.md docs/Docker.md docs/INSTALL.md docs/Office365.md docs/USAGE.md
+DOCFILES = README.md config LICENSE CHANGELOG.md docs/Docker.md docs/INSTALL.md docs/Office365.md docs/USAGE.md docs/BusinessSharedFolders.md
 
 ifneq ("$(wildcard /etc/redhat-release)","")
 RHEL = $(shell cat /etc/redhat-release | grep -E "(Red Hat Enterprise Linux Server|CentOS)" | wc -l)

README.md

@@ -17,8 +17,8 @@ This client is a 'fork' of the [skilion](https://github.com/skilion/onedrive) cl
 *   File upload / download validation to ensure data integrity
 *   Resumable uploads
 *   Support OneDrive for Business (part of Office 365)
-*   Shared folders (OneDrive Personal)
-*   SharePoint / Office 365 Shared Libraries
+*   Shared Folder support for OneDrive Personal and OneDrive Business accounts
+*   SharePoint / Office365 Shared Libraries
 *   Desktop notifications via libnotify
 *   Dry-run capability to test configuration changes
 *   Prevent major OneDrive accidental data deletion after configuration change
@@ -37,6 +37,9 @@ See [docs/USAGE.md](https://github.com/abraunegg/onedrive/blob/master/docs/USAGE
 ## Docker support
 See [docs/Docker.md](https://github.com/abraunegg/onedrive/blob/master/docs/Docker.md)
 
+## OneDrive Business Shared Folders
+See [docs/BusinessSharedFolders.md](https://github.com/abraunegg/onedrive/blob/master/docs/docs/BusinessSharedFolders.md)
+
 ## SharePoint / Office 365 Shared Libraries (Business or Education)
 See [docs/Office365.md](https://github.com/abraunegg/onedrive/blob/master/docs/Office365.md)
 

config

@@ -37,3 +37,5 @@
 # application_id = ""
 # resync = "false"
 # bypass_data_preservation = "false"
+# azure_ad_endpoint = ""
+# sync_business_shared_folders = "false"

docs/BusinessSharedFolders.md

@@ -0,0 +1,187 @@
+# How to configure OneDrive Business Shared Folder Sync
+Syncing OneDrive Business Shared Folders requires additional configuration for your 'onedrive' client:
+1.  List available shared folders to determine which folder you wish to sync & to validate that you have access to that folder
+2.  Create a new file called 'business_shared_folders' in your config directory which contains a list of the shared folders you wish to sync
+3.  Perform a sync
+
+## Listing available OneDrive Business Shared Folders
+List the available OneDrive Business Shared folders with the following command:
+```text
+onedrive --list-shared-folders
+```
+   This will return a listing of all OneDrive Business Shared folders which have been shared with you and by whom. This is important for conflict resolution:
+```text
+Initializing the Synchronization Engine ...
+
+Listing available OneDrive Business Shared Folders:
+---------------------------------------
+Shared Folder:   SharedFolder0
+Shared By:       Firstname Lastname
+---------------------------------------
+Shared Folder:   SharedFolder1
+Shared By:       Firstname Lastname
+---------------------------------------
+Shared Folder:   SharedFolder2
+Shared By:       Firstname Lastname
+---------------------------------------
+Shared Folder:   SharedFolder0
+Shared By:       Firstname Lastname (user@domain)
+---------------------------------------
+Shared Folder:   SharedFolder1
+Shared By:       Firstname Lastname (user@domain)
+---------------------------------------
+Shared Folder:   SharedFolder2
+Shared By:       Firstname Lastname (user@domain)
+...
+```
+
+## Configuring OneDrive Business Shared Folders
+1.  Create a new file called 'business_shared_folders' in your config directory
+2.  On each new line, list the OneDrive Business Shared Folder you wish to sync
+```text
+[alex@centos7full onedrive]$ cat ~/.config/onedrive/business_shared_folders
+# comment
+Child Shared Folder
+# Another comment
+Top Level to Share
+[alex@centos7full onedrive]$ 
+```
+3.  Validate your configuration with `onedrive --display-config`:
+```text
+Configuration file successfully loaded
+onedrive version                       = v2.4.3
+Config path                            = /home/alex/.config/onedrive-business/
+Config file found in config path       = true
+Config option 'check_nosync'           = false
+Config option 'sync_dir'               = /home/alex/OneDriveBusiness
+Config option 'skip_dir'               = 
+Config option 'skip_file'              = ~*|.~*|*.tmp
+Config option 'skip_dotfiles'          = false
+Config option 'skip_symlinks'          = false
+Config option 'monitor_interval'       = 300
+Config option 'min_notify_changes'     = 5
+Config option 'log_dir'                = /var/log/onedrive/
+Config option 'classify_as_big_delete' = 1000
+Config option 'sync_root_files'        = false
+Selective sync 'sync_list' configured  = false
+Business Shared Folders configured     = true
+business_shared_folders contents:
+# comment
+Child Shared Folder
+# Another comment
+Top Level to Share
+```
+
+## Performing a sync of OneDrive Business Shared Folders
+Perform a standalone sync using the following command: `onedrive --synchronize --sync-shared-folders --verbose`:
+```text
+onedrive --synchronize --sync-shared-folders --verbose
+Using 'user' Config Dir: /home/alex/.config/onedrive-business/
+Using 'system' Config Dir: 
+Configuration file successfully loaded
+Initializing the OneDrive API ...
+Configuring Global Azure AD Endpoints
+Opening the item database ...
+All operations will be performed in: /home/alex/OneDriveBusiness
+Application version: v2.4.3
+Account Type: business
+Default Drive ID: b!bO8V7s9SSk6r7mWHpIjURotN33W1W2tEv3OXV_oFIdQimEdOHR-1So7CqeT1MfHA
+Default Root ID: 01WIXGO5V6Y2GOVW7725BZO354PWSELRRZ
+Remaining Free Space: 1098316220277
+Fetching details for OneDrive Root
+OneDrive Root exists in the database
+Initializing the Synchronization Engine ...
+Syncing changes from OneDrive ...
+Applying changes of Path ID: 01WIXGO5V6Y2GOVW7725BZO354PWSELRRZ
+Number of items from OneDrive to process: 0
+Attempting to sync OneDrive Business Shared Folders
+Syncing this OneDrive Business Shared Folder: Child Shared Folder
+OneDrive Business Shared Folder - Shared By:  test user
+Applying changes of Path ID: 01JRXHEZMREEB3EJVHNVHKNN454Q7DFXPR
+Adding OneDrive root details for processing
+Adding OneDrive folder details for processing
+Adding 4 OneDrive items for processing from OneDrive folder
+Adding 2 OneDrive items for processing from /Child Shared Folder/Cisco VDI Whitepaper
+Adding 2 OneDrive items for processing from /Child Shared Folder/SMPP_Shared
+Processing 11 OneDrive items to ensure consistent local state
+Syncing this OneDrive Business Shared Folder: Top Level to Share
+OneDrive Business Shared Folder - Shared By:  test user ([email protected])
+Applying changes of Path ID: 01JRXHEZLRMXHKBYZNOBF3TQOPBXS3VZMA
+Adding OneDrive root details for processing
+Adding OneDrive folder details for processing
+Adding 4 OneDrive items for processing from OneDrive folder
+Adding 3 OneDrive items for processing from /Top Level to Share/10-Files
+Adding 2 OneDrive items for processing from /Top Level to Share/10-Files/Cisco VDI Whitepaper
+Adding 2 OneDrive items for processing from /Top Level to Share/10-Files/Images
+Adding 8 OneDrive items for processing from /Top Level to Share/10-Files/Images/JPG
+Adding 8 OneDrive items for processing from /Top Level to Share/10-Files/Images/PNG
+Adding 2 OneDrive items for processing from /Top Level to Share/10-Files/SMPP
+Processing 31 OneDrive items to ensure consistent local state
+Uploading differences of ~/OneDriveBusiness
+Processing root
+The directory has not changed
+Processing SMPP_Local
+The directory has not changed
+Processing SMPP-IF-SPEC_v3_3-24858.pdf
+The file has not changed
+Processing SMPP_v3_4_Issue1_2-24857.pdf
+The file has not changed
+Processing new_local_file.txt
+The file has not changed
+Processing root
+The directory has not changed
+...
+The directory has not changed
+Processing week02-03-Combinational_Logic-v1.pptx
+The file has not changed
+Uploading new items of ~/OneDriveBusiness
+Applying changes of Path ID: 01WIXGO5V6Y2GOVW7725BZO354PWSELRRZ
+Number of items from OneDrive to process: 0
+Attempting to sync OneDrive Business Shared Folders
+Syncing this OneDrive Business Shared Folder: Child Shared Folder
+OneDrive Business Shared Folder - Shared By:  test user
+Applying changes of Path ID: 01JRXHEZMREEB3EJVHNVHKNN454Q7DFXPR
+Adding OneDrive root details for processing
+Adding OneDrive folder details for processing
+Adding 4 OneDrive items for processing from OneDrive folder
+Adding 2 OneDrive items for processing from /Child Shared Folder/Cisco VDI Whitepaper
+Adding 2 OneDrive items for processing from /Child Shared Folder/SMPP_Shared
+Processing 11 OneDrive items to ensure consistent local state
+Syncing this OneDrive Business Shared Folder: Top Level to Share
+OneDrive Business Shared Folder - Shared By:  test user ([email protected])
+Applying changes of Path ID: 01JRXHEZLRMXHKBYZNOBF3TQOPBXS3VZMA
+Adding OneDrive root details for processing
+Adding OneDrive folder details for processing
+Adding 4 OneDrive items for processing from OneDrive folder
+Adding 3 OneDrive items for processing from /Top Level to Share/10-Files
+Adding 2 OneDrive items for processing from /Top Level to Share/10-Files/Cisco VDI Whitepaper
+Adding 2 OneDrive items for processing from /Top Level to Share/10-Files/Images
+Adding 8 OneDrive items for processing from /Top Level to Share/10-Files/Images/JPG
+Adding 8 OneDrive items for processing from /Top Level to Share/10-Files/Images/PNG
+Adding 2 OneDrive items for processing from /Top Level to Share/10-Files/SMPP
+Processing 31 OneDrive items to ensure consistent local state
+```
+
+**Note:** Whenever you modify the `business_shared_folders` file you must perform a `--resync` of your database to clean up stale entries due to changes in your configuration.
+
+## Enable / Disable syncing of OneDrive Business Shared Folders
+Performing a sync of the configured OneDrive Business Shared Folders can be enabled / disabled via adding the following to your configuration file.
+
+### Enable syncing of OneDrive Business Shared Folders via config file
+```text
+sync_business_shared_folders = "true"
+```
+
+### Disable syncing of OneDrive Business Shared Folders via config file
+```text
+sync_business_shared_folders = "false"
+```
+
+## Known Issues
+Shared folders, shared with you from people outside of your 'organisation' are unable to be synced. This is due to the Microsoft Graph API not presenting these folders.
+
+Shared folders that match this scenario, when you view 'Shared' via OneDrive online, will have a 'world' symbol as per below:
+
+![shared_with_me](./images/shared_with_me.jpg)
+
+This issue is being tracked by: [#966](https://github.com/abraunegg/onedrive/issues/966)

src/config.d

@@ -23,6 +23,7 @@ final class Config
 	public string configFileSyncDir = "";
 	public string configFileSkipFile = "";
 	public string configFileSkipDir = "";
+	public string businessSharedFolderFilePath = "";
 	private string userConfigFilePath = "";
 	private string systemConfigFilePath = "";
 	// was the application just authorised - paste of response uri
@@ -33,8 +34,9 @@ final class Config
 	private string[string] stringValues;
 	private bool[string] boolValues;
 	private long[string] longValues;
+	// Compile time regex - this does not change
 	public auto configRegex = ctRegex!(`^(\w+)\s*=\s*"(.*)"\s*$`);
-
+	
 	this(string confdirOption)
 	{
 		// default configuration - entries in config file ~/.config/onedrive/config
@@ -100,6 +102,8 @@ final class Config
 		//     AD Endpoint:    https://login.chinacloudapi.cn
 		//     Graph Endpoint: 	https://microsoftgraph.chinacloudapi.cn
 		stringValues["azure_ad_endpoint"] = "";
+		// Allow enable / disable of the syncing of OneDrive Business Shared Folders via configuration file
+		boolValues["sync_business_shared_folders"] = false;
 
 		// DEVELOPER OPTIONS 
 		// display_memory = true | false
@@ -189,6 +193,7 @@ final class Config
 		userConfigFilePath = buildNormalizedPath(configDirName ~ "/config");
 		syncListFilePath = buildNormalizedPath(configDirName ~ "/sync_list");
 		systemConfigFilePath = buildNormalizedPath(systemConfigDirName ~ "/config");
+		businessSharedFolderFilePath = buildNormalizedPath(configDirName ~ "/business_shared_folders");
 
 		// Debug Output for application set variables based on configDirName
 		log.vdebug("refreshTokenFilePath = ", refreshTokenFilePath);
@@ -199,6 +204,7 @@ final class Config
 		log.vdebug("userConfigFilePath = ", userConfigFilePath);
 		log.vdebug("syncListFilePath = ", syncListFilePath);
 		log.vdebug("systemConfigFilePath = ", systemConfigFilePath);
+		log.vdebug("businessSharedFolderFilePath = ", businessSharedFolderFilePath);
 	}
 
 	bool initialize()
@@ -259,13 +265,16 @@ final class Config
 		boolValues["force"]               = false;
 		boolValues["remove_source_files"] = false;
 		boolValues["skip_dir_strict_match"] = false;
-
+		boolValues["list_business_shared_folders"] = false;
+
 		// Application Startup option validation
 		try {
 			string tmpStr;
 			bool tmpBol;
 			long tmpVerb;
+			// duplicated from main.d to get full help output!
 			auto opt = getopt(
+
 				args,
 				std.getopt.config.bundling,
 				std.getopt.config.caseSensitive,
@@ -404,7 +413,6 @@ final class Config
 				"user-agent",
 					"Specify a User Agent string to the http client",
 					&stringValues["user_agent"],
-				// duplicated from main.d to get full help output!
 				"confdir",
 					"Set the directory used to store the configuration files",
 					&tmpStr,
@@ -413,7 +421,13 @@ final class Config
 					&tmpVerb,
 				"version",
 					"Print the version and exit",
-					&tmpBol
+					&tmpBol,
+				"list-shared-folders",
+					"List OneDrive Business Shared Folders",
+					&boolValues["list_business_shared_folders"],
+				"sync-shared-folders",
+					"Sync OneDrive Business Shared Folders",
+					&boolValues["sync_business_shared_folders"]
 			);
 			if (opt.helpWanted) {
 				outputLongHelp(opt.options);

src/itemdb.d

@@ -2,6 +2,8 @@ import std.datetime;
 import std.exception;
 import std.path;
 import std.string;
+import std.stdio;
+import std.algorithm.searching;
 import core.stdc.stdlib;
 import sqlite;
 static import log;
@@ -368,9 +370,14 @@ final class ItemDatabase
 					if (r2.empty) {
 						// root reached
 						assert(path.length >= 4);
-						// remove "root"
-						if (path.length >= 5) path = path[5 .. $];
-						else path = path[4 .. $];
+						// remove "root/" from path string if it exists
+						if (path.length >= 5) {
+							if (canFind(path, "root/")){
+								path = path[5 .. $];
+							}
+						} else {
+							path = path[4 .. $];
+						}
 						// special case of computing the path of the root itself
 						if (path.length == 0) path = ".";
 						break;
@@ -427,17 +434,39 @@ final class ItemDatabase
 	// As we query /children to get all children from OneDrive, update anything in the database 
 	// to be flagged as not-in-sync, thus, we can use that flag to determing what was previously
 	// in-sync, but now deleted on OneDrive
-	void downgradeSyncStatusFlag()
+	void downgradeSyncStatusFlag(const(char)[] driveId, const(char)[] id)
 	{
-		db.exec("UPDATE item SET syncStatus = 'N'");
+		assert(driveId);
+		auto stmt = db.prepare("UPDATE item SET syncStatus = 'N' WHERE driveId = ?1 AND id = ?2");
+		stmt.bind(1, driveId);
+		stmt.bind(2, id);
+		stmt.exec();
 	}
 
 	// National Cloud Deployments (US and DE) do not support /delta as a query
 	// Select items that have a out-of-sync flag set
-	Item[] selectOutOfSyncItems()
+	Item[] selectOutOfSyncItems(const(char)[] driveId)
 	{
+		assert(driveId);
 		Item[] items;
-		auto stmt = db.prepare("SELECT * FROM item WHERE syncStatus = 'N'");
+		auto stmt = db.prepare("SELECT * FROM item WHERE syncStatus = 'N' AND driveId = ?1");
+		stmt.bind(1, driveId);
+		auto res = stmt.exec();
+		while (!res.empty) {
+			items ~= buildItem(res);
+			res.step();
+		}
+		return items;
+	}
+
+	// OneDrive Business Folders are stored in the database potentially without a root | parentRoot link
+	// Select items associated with the provided driveId
+	Item[] selectByDriveId(const(char)[] driveId)
+	{
+		assert(driveId);
+		Item[] items;
+		auto stmt = db.prepare("SELECT * FROM item WHERE driveId = ?1 AND parentId IS NULL");
+		stmt.bind(1, driveId);
 		auto res = stmt.exec();
 		while (!res.empty) {
 			items ~= buildItem(res);