WIP: Sync ODB shared folders from external organisation

docs/BusinessSharedFolders.md

@@ -177,12 +177,3 @@ sync_business_shared_folders = "true"
 ```text
 sync_business_shared_folders = "false"
 ```
-
-## Known Issues
-Shared folders, shared with you from people outside of your 'organisation' are unable to be synced. This is due to the Microsoft Graph API not presenting these folders.
-
-Shared folders that match this scenario, when you view 'Shared' via OneDrive online, will have a 'world' symbol as per below:
-
-![shared_with_me](./images/shared_with_me.JPG)
-
-This issue is being tracked by: [#966](https://github.com/abraunegg/onedrive/issues/966)

src/onedrive.d

@@ -66,6 +66,9 @@ private {
 	// What is 'shared with me' Query
 	string sharedWithMe = globalGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
 
+	// What are my 'tenant' details
+	string sharepointTenantId = globalGraphEndpoint ~ "/v1.0/organization";
+
 	// Item Queries
 	string itemByIdUrl = globalGraphEndpoint ~ "/v1.0/me/drive/items/";
 	string itemByPathUrl = globalGraphEndpoint ~ "/v1.0/me/drive/root:/";
@@ -105,8 +108,16 @@ final class OneDriveApi
 	private SysTime accessTokenExpiration;
 	private HTTP http;
 
-	// if true, every new access token is printed
+	// If true, every new access token is printed
 	bool printAccessToken;
+
+	// OneDrive Business External Shared Folder tenant handling
+	bool externalTenant = false;
+	string externalTenantID = "";
+	string externalTokenUrl = "";
+	string externalRefreshToken = "";
+	string externalAccessToken = "";
+	SysTime externalAccessTokenExpiration;
 
 	this(Config cfg)
 	{
@@ -196,6 +207,7 @@ final class OneDriveApi
 				siteDriveUrl = usl4GraphEndpoint ~ "/v1.0/sites/";
 				// Shared With Me
 				sharedWithMe = usl4GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
+				sharepointTenantId = usl4GraphEndpoint ~ "/v1.0/organization"; 
 				break;
 			case "USL5":
 				log.log("Configuring Azure AD for US Government Endpoints (DOD)");
@@ -222,6 +234,7 @@ final class OneDriveApi
 				siteDriveUrl = usl5GraphEndpoint ~ "/v1.0/sites/";
 				// Shared With Me
 				sharedWithMe = usl5GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
+				sharepointTenantId = usl5GraphEndpoint ~ "/v1.0/organization";
 				break;
 			case "DE":
 				log.log("Configuring Azure AD Germany");
@@ -248,6 +261,7 @@ final class OneDriveApi
 				siteDriveUrl = deGraphEndpoint ~ "/v1.0/sites/";
 				// Shared With Me
 				sharedWithMe = deGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
+				sharepointTenantId = deGraphEndpoint ~ "/v1.0/organization";
 				break;
 			case "CN":
 				log.log("Configuring AD China operated by 21Vianet");
@@ -274,6 +288,7 @@ final class OneDriveApi
 				siteDriveUrl = cnGraphEndpoint ~ "/v1.0/sites/";
 				// Shared With Me
 				sharedWithMe = cnGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
+				sharepointTenantId = cnGraphEndpoint ~ "/v1.0/organization";
 				break;
 			// Default - all other entries 
 			default:
@@ -379,7 +394,7 @@ final class OneDriveApi
 	{
 		import std.stdio, std.regex;
 		char[] response;
-		string url = authUrl ~ "?client_id=" ~ clientId ~ "&scope=Files.ReadWrite%20Files.ReadWrite.all%20Sites.Read.All%20Sites.ReadWrite.All%20offline_access&response_type=code&redirect_uri=" ~ redirectUrl;
+		string url = authUrl ~ "?client_id=" ~ clientId ~ "&scope=User.Read%20Files.ReadWrite%20Files.ReadWrite.all%20Sites.Read.All%20Sites.ReadWrite.All%20offline_access&response_type=code&redirect_uri=" ~ redirectUrl;
 		string authFilesString = cfg.getValueString("auth_files");
 		if (authFilesString == "") {
 			log.log("Authorize this app visiting:\n");
@@ -438,6 +453,50 @@ final class OneDriveApi
 		.retryAfterValue = 0;
 	}
 
+	void setExternalTenant(string externalTenantValue)
+	{
+		// Flag that we are using an external tenant
+		externalTenant = true;
+		externalTenantID = externalTenantValue;
+
+		// Configure externalTokenUrl for this external tenant
+		string azureConfigValue = cfg.getValueString("azure_ad_endpoint");
+		switch(azureConfigValue) {
+			case "":
+				externalTokenUrl = globalAuthEndpoint ~ "/" ~ externalTenantID ~ "/oauth2/v2.0/token";
+				break;
+			case "USL4":
+				externalTokenUrl = usl4AuthEndpoint ~ "/" ~ externalTenantID ~ "/oauth2/v2.0/token";
+				break;
+			case "USL5":
+				externalTokenUrl = usl5AuthEndpoint ~ "/" ~ externalTenantID ~ "/oauth2/v2.0/token";
+				break;
+			case "DE":
+				externalTokenUrl = deAuthEndpoint ~ "/" ~ externalTenantID ~ "/oauth2/v2.0/token";
+				break;
+			case "CN":
+				externalTokenUrl = cnAuthEndpoint ~ "/" ~ externalTenantID ~ "/oauth2/v2.0/token";
+				tokenUrl = cnAuthEndpoint ~ "/common/oauth2/v2.0/token";
+				break;
+			// Default - all other entries 
+			default:
+				externalTokenUrl = globalAuthEndpoint ~ "/" ~ externalTenantID ~ "/oauth2/v2.0/token";
+		}
+
+		// Get new token and configure this for use
+		newToken();	
+	}
+
+	void clearExternalTenant()
+	{
+		// Clear any external tenant that was set
+		externalTenant = false;
+		externalTenantID = "";
+		externalTokenUrl = "";
+		externalRefreshToken = "";
+		externalAccessToken = "";
+	}
+
 	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
 	JSONValue getDefaultDrive()
 	{
@@ -465,11 +524,23 @@ final class OneDriveApi
 		return get(url);
 	}
 
+	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/site_get
+	JSONValue getTenantID()
+	{
+		checkAccessTokenExpired();
+		const(char)[] url;
+		url = sharepointTenantId;
+		return get(url);
+	}
+
 	// https://docs.microsoft.com/en-us/graph/api/drive-sharedwithme
 	JSONValue getSharedWithMe()
 	{
 		checkAccessTokenExpired();
-		return get(sharedWithMe);
+		const(char)[] url;
+		url = sharedWithMe;
+		url ~= "?allowexternal=true";
+		return get(url);
 	}
 
 	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
@@ -651,7 +722,7 @@ final class OneDriveApi
 		return get(url);
 	}
 
-	// Return the requested details of the specified id
+	// Return the requested details of the specified item id
 	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
 	JSONValue getFileDetails(const(char)[] driveId, const(char)[] id)
 	{
@@ -775,38 +846,60 @@ final class OneDriveApi
 	private void acquireToken(const(char)[] postData)
 	{
 		JSONValue response;
-
 		try {
-			response = post(tokenUrl, postData);
+			if (!externalTenant) {
+				// use normal token url
+				response = post(tokenUrl, postData);
+			} else {
+				// use external tenant token url
+				response = post(externalTokenUrl, postData);
+			}
 		} catch (OneDriveException e) {
 			// an error was generated
 			displayOneDriveErrorMessage(e.msg);
 		}
 
 		if (response.type() == JSONType.object) {
-			if ("access_token" in response){
-				accessToken = "bearer " ~ response["access_token"].str();
-				refreshToken = response["refresh_token"].str();
-				accessTokenExpiration = Clock.currTime() + dur!"seconds"(response["expires_in"].integer());
-				if (!.dryRun) {
-					try {
-						// try and update the refresh_token file
-						std.file.write(cfg.refreshTokenFilePath, refreshToken);
-						log.vdebug("Setting file permissions for: ", cfg.refreshTokenFilePath);
-						cfg.refreshTokenFilePath.setAttributes(cfg.returnRequiredFilePermisions());
-					} catch (FileException e) {
-						// display the error message
-						displayFileSystemErrorMessage(e.msg);
+			// what sort of tenant token are we processing?
+			if (!externalTenant) {
+				// Normal tenant / token processing
+				if ("access_token" in response){
+					accessToken = "bearer " ~ response["access_token"].str();
+					refreshToken = response["refresh_token"].str();
+					accessTokenExpiration = Clock.currTime() + dur!"seconds"(response["expires_in"].integer());
+					if (!.dryRun) {
+						try {
+							// try and update the refresh_token file
+							log.vdebug("Updating refresh_token file with new token from OneDrive");
+							std.file.write(cfg.refreshTokenFilePath, refreshToken);
+							log.vdebug("Setting file permissions for: ", cfg.refreshTokenFilePath);
+							cfg.refreshTokenFilePath.setAttributes(cfg.returnRequiredFilePermisions());
+						} catch (FileException e) {
+							// display the error message
+							displayFileSystemErrorMessage(e.msg);
+						}
 					}
+					if (printAccessToken) writeln("New access token: ", accessToken);
+				} else {
+					log.error("\nInvalid authentication response from OneDrive. Please check the response uri\n");
+					// re-authorize
+					authorize();
 				}
-				if (printAccessToken) writeln("New access token: ", accessToken);
 			} else {
-				log.error("\nInvalid authentication response from OneDrive. Please check the response uri\n");
-				// re-authorize
-				authorize();
+				// External tenant token handling
+				if ("access_token" in response){
+					log.vdebug("Updating access token with new token received from External OneDrive 3rd Party");
+					externalAccessToken = "bearer " ~ response["access_token"].str();
+					log.vdebug("Updating refresh_token with new token received from External OneDrive 3rd Party");
+					externalRefreshToken = response["refresh_token"].str();
+					externalAccessTokenExpiration = Clock.currTime() + dur!"seconds"(response["expires_in"].integer());
+					if (printAccessToken) writeln("New external access token: ", externalAccessToken);
+				} else {
+					log.error("\nInvalid authentication response from OneDrive. Please check the response uri\n");
+				}
 			}
 		} else {
-			log.vdebug("Invalid JSON response from OneDrive unable to initialize application");
+			log.vdebug("Invalid JSON response from OneDrive unable to aquire token to initialize application");
 		}
 	}
 
@@ -828,7 +921,20 @@ final class OneDriveApi
 
 	private void addAccessTokenHeader()
 	{
-		http.addRequestHeader("Authorization", accessToken);
+		// Which access token header must we add?
+		if (!externalTenant) {
+			// Add the default client accessToken
+			http.addRequestHeader("Authorization", accessToken);
+		} else {
+			// Add the external tenant accessToken
+			if (externalAccessToken.empty) {
+				// if this is still empty, we are probably in the process of getting an updated external access token
+				// use our original one for the moment until this is set
+				http.addRequestHeader("Authorization", accessToken);
+			} else {
+				http.addRequestHeader("Authorization", externalAccessToken);
+			}
+		}
 	}
 
 	private JSONValue get(const(char)[] url, bool skipToken = false)

src/sync.d

@@ -596,11 +596,24 @@ final class SyncEngine
 
 		// Check OneDrive Business Shared Folders, if configured to do so
 		if (syncBusinessFolders){
-			// query OneDrive Business Shared Folders shared with me
+			// Get My Tenent Details
+			string myTenantID;
+			JSONValue tenantDetailsResponse = onedrive.getTenantID();
+			if (tenantDetailsResponse.type() == JSONType.object) {
+				foreach (searchResult; tenantDetailsResponse["value"].array) {
+					myTenantID = searchResult["id"].str;
+				}
+			} else {
+				// Log that an invalid JSON object was returned
+				log.error("ERROR: onedrive.getTenantID call returned an invalid JSON Object");
+			}
+
+			// Query OneDrive Business Shared Folders shared with me
 			log.vlog("Attempting to sync OneDrive Business Shared Folders");
 			JSONValue graphQuery = onedrive.getSharedWithMe();
 			if (graphQuery.type() == JSONType.object) {
 				string sharedFolderName;
+				bool isExternalTenant = false;
 				foreach (searchResult; graphQuery["value"].array) {
 					// Configure additional logging items for this array element
 					string sharedByName;
@@ -626,16 +639,25 @@ final class SyncEngine
 							bool itemInDatabase = false;
 							bool itemLocalDirExists = false;
 							bool itemPathIsLocal = false;
+							isExternalTenant = false;
 
 							// "what if" there are 2 or more folders shared with me have the "same" name?
 							// The folder name will be the same, but driveId will be different
 							// This will then cause these 'shared folders' to cross populate data, which may not be desirable
 							log.vdebug("Shared Folder Name: ", sharedFolderName);
 							log.vdebug("Parent Drive Id:    ", searchResult["remoteItem"]["parentReference"]["driveId"].str);
 							log.vdebug("Shared Item Id:     ", searchResult["remoteItem"]["id"].str);
-							Item databaseItem;
 
-							// for each driveid in the existing driveIDsArray 
+							// Is this OneDrive Shared Folder on an external tenant?
+							if (searchResult["remoteItem"]["sharepointIds"]["tenantId"].str != myTenantID) {
+								isExternalTenant = true;
+								log.vdebug("This shared folder is shared from an external organisation as the tenant is different");
+								// Have to configure the access to this tenant, which requires separate tokenUrl for that tenant
+								onedrive.setExternalTenant(searchResult["remoteItem"]["sharepointIds"]["tenantId"].str);
+							}
+
+							// for each driveid in the existing driveIDsArray
+							Item databaseItem;
 							foreach (searchDriveId; driveIDsArray) {
 								log.vdebug("searching database for: ", searchDriveId, " ", sharedFolderName);
 								if (itemdb.selectByPath(sharedFolderName, searchDriveId, databaseItem)) {
@@ -709,7 +731,7 @@ final class SyncEngine
 									}
 								}
 							}	
-						}
+						}	
 					} else {
 						// not a folder, is this a file?
 						if (isItemFile(searchResult)) {
@@ -731,6 +753,12 @@ final class SyncEngine
 							log.log("WARNING: Not syncing this OneDrive Business Shared item: ", searchResult["name"].str);
 						}
 					}
+
+					// Was this shared folder on an external tenant?
+					if (isExternalTenant) {
+						// clear external tenant
+						onedrive.clearExternalTenant();
+					}					
 				}
 			} else {
 				// Log that an invalid JSON object was returned
@@ -6107,6 +6135,19 @@ final class SyncEngine
 	{
 		// List OneDrive Business Shared Folders
 		log.log("\nListing available OneDrive Business Shared Folders:");
+
+		// Get My Tenent Details
+		string myTenantID;
+		JSONValue tenantDetailsResponse = onedrive.getTenantID();
+		if (tenantDetailsResponse.type() == JSONType.object) {
+			foreach (searchResult; tenantDetailsResponse["value"].array) {
+				myTenantID = searchResult["id"].str;
+			}
+		} else {
+			// Log that an invalid JSON object was returned
+			log.error("ERROR: onedrive.getTenantID call returned an invalid JSON Object");
+		}
+
 		// Query the GET /me/drive/sharedWithMe API
 		JSONValue graphQuery = onedrive.getSharedWithMe();
 		if (graphQuery.type() == JSONType.object) {
@@ -6140,6 +6181,7 @@ final class SyncEngine
 						}
 						// Output query result
 						log.log("---------------------------------------");
+						// Default output
 						log.log("Shared Folder:   ", sharedFolderName);
 						if ((sharedByName != "") && (sharedByEmail != "")) {
 							log.log("Shared By:       ", sharedByName, " (", sharedByEmail, ")");
@@ -6148,11 +6190,20 @@ final class SyncEngine
 								log.log("Shared By:       ", sharedByName);
 							}
 						}
+						// Tenant details
+						if (searchResult["remoteItem"]["sharepointIds"]["tenantId"].str == myTenantID) {
+							log.log("External Organisation: no");
+						} else {
+							log.log("External Organisation: yes");
+						}
+
+						// Extra verbose output
 						log.vlog("Item Id:         ", searchResult["remoteItem"]["id"].str);
 						log.vlog("Parent Drive Id: ", searchResult["remoteItem"]["parentReference"]["driveId"].str);
 						if ("id" in searchResult["remoteItem"]["parentReference"]) {
 							log.vlog("Parent Item Id:  ", searchResult["remoteItem"]["parentReference"]["id"].str);
 						}
+						log.vlog("Tenant ID:             ", searchResult["remoteItem"]["sharepointIds"]["tenantId"].str);
 					}
 				}
 			}