Server: Resolves #9355: Add sync debug page to Joplin Server

.eslintignore

@@ -1003,6 +1003,7 @@ packages/renderer/index.js
 packages/renderer/noteStyle.js
 packages/renderer/pathUtils.js
 packages/renderer/utils.js
+packages/server/public/js/index/sync_debug.js
 packages/tools/build-release-stats.test.js
 packages/tools/build-release-stats.js
 packages/tools/build-translation.js

.gitignore

@@ -985,6 +985,7 @@ packages/renderer/index.js
 packages/renderer/noteStyle.js
 packages/renderer/pathUtils.js
 packages/renderer/utils.js
+packages/server/public/js/index/sync_debug.js
 packages/tools/build-release-stats.test.js
 packages/tools/build-release-stats.js
 packages/tools/build-translation.js

packages/server/gulpfile.js

@@ -1,11 +1,37 @@
 const gulp = require('gulp');
 const utils = require('@joplin/tools/gulp/utils');
 const compilePackageInfo = require('@joplin/tools/compilePackageInfo');
+const { execCommand } = require('@joplin/utils');
 const fs = require('fs-extra');
+const path = require('path');
 
 const distDir = `${__dirname}/dist`;
 
+const tsConfigPaths = [
+	path.join(__dirname, 'tsconfig.json'),
+	path.join(__dirname, 'public', 'js', 'tsconfig.json'),
+];
+
 const tasks = {
+	tsc: {
+		fn: async () => {
+			for (const tsConfigPath of tsConfigPaths) {
+				await execCommand(['tsc', '--project', tsConfigPath]);
+			}
+		},
+	},
+
+	watch: {
+		fn: async () => {
+			const watchTasks = tsConfigPaths.map(tsConfigPath => {
+				return execCommand(
+					['tsc', '--watch', '--preserveWatchOutput', '--project', tsConfigPath],
+				);
+			});
+			await Promise.all(watchTasks);
+		},
+	},
+
 	compilePackageInfo: {
 		fn: async () => {
 			await fs.mkdirp(distDir);

packages/server/package.json

@@ -12,14 +12,14 @@
     "devDropDb": "node dist/app.js --env dev --drop-db",
     "start": "node dist/app.js",
     "generateTypes": "rm -f db-buildTypes.sqlite && yarn run start --env buildTypes migrate latest && node dist/tools/generateTypes.js && mv db-buildTypes.sqlite schema.sqlite",
-    "tsc": "tsc --project tsconfig.json",
+    "tsc": "gulp tsc",
     "test": "jest --verbose=false",
     "test-ci": "yarn test",
     "test-debug": "node --inspect node_modules/.bin/jest -- --verbose=false",
     "clean": "gulp clean",
     "populateDatabase": "JOPLIN_TESTS_SERVER_DB=pg node dist/utils/testing/populateDatabase",
     "stripeListen": "stripe listen --forward-to http://joplincloud.local:22300/stripe/webhook",
-    "watch": "tsc --watch --preserveWatchOutput --project tsconfig.json"
+    "watch": "gulp watch"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "3.296.0",

packages/server/public/css/index/sync_debug.css

@@ -0,0 +1,37 @@
+
+
+#debug-tools-container {
+	--border-color: #444;
+}
+
+#debug-tools-container legend {
+	font-weight: bold;
+}
+
+#debug-tools-container fieldset {
+	border-left: 1px solid var(--border-color);
+	padding-left: 5px;
+	margin-top: 14px;
+	/* Required for proper overflow. See
+	   https://stackoverflow.com/a/29499408 */
+	min-width: 0;
+}
+
+#debug-tools-container .output {
+	border: 1px solid var(--border-color);
+	border-radius: 5px;
+	margin: 5px;
+	padding: 5px;
+}
+
+#debug-tools-container .output pre {
+	white-space: pre-wrap;
+	word-wrap: break-word;
+	max-height: 500px;
+	overflow-y: auto;
+	border-radius: 5px;
+}
+
+#debug-tools-container .output.empty {
+	display: none;
+}

packages/server/public/js/index/sync_debug.ts

@@ -0,0 +1,157 @@
+
+const itemLinkToId = (link: string) => {
+	// Examples of supported links:
+	// - joplin://x-callback-url/openFolder?id=6c8caeec01a34c0f95487a04ebb79cb9
+	// - joplin://x-callback-url/openNote?id=6c8caeec01a34c0f95487a04ebb79cb9
+	// - :/6c8caeec01a34c0f95487a04ebb79cb9
+	// - /home/user/.config/joplin-desktop/resources/6c8caeec01a34c0f95487a04ebb79cb9.svg
+	// - [title](:/6c8caeec01a34c0f95487a04ebb79cb9)
+	const linkRegexs = [
+		// External item
+		/^joplin:\/\/x-callback-url\/(?:openFolder|openNote)\?id=(\w+)$/,
+
+		// Internal links
+		/^\/:(\w+)$/,
+		/^!?\[.*\]\(:\/(\w+)\)$/,
+
+		// Resource file URLs
+		/^(?:file:\/\/)?.*[/\\]resources[/\\](\w+)\.\w+$/,
+	];
+
+	for (const regex of linkRegexs) {
+		const match = regex.exec(link);
+		if (match) {
+			return match[1];
+		}
+	}
+
+	return link;
+};
+
+type OnItemCheckerSubmitCallback = (
+	itemId: string, outputHeading: HTMLElement, outputDetails: HTMLElement,
+)=> Promise<void>;
+
+const setUpItemChecker = (parent: HTMLElement, onSubmit: OnItemCheckerSubmitCallback) => {
+	const button = document.createElement('button');
+	button.innerText = 'Submit';
+
+	const input = parent.querySelector('input');
+	const outputContainer = document.createElement('div');
+	outputContainer.classList.add('output', 'empty');
+
+	const outputHeading = document.createElement('h3');
+	const outputDetailsContainer = document.createElement('details');
+	const outputDetailsContent = document.createElement('pre');
+
+	outputHeading.setAttribute('aria-live', 'polite');
+
+	outputDetailsContainer.appendChild(outputDetailsContent);
+	outputContainer.replaceChildren(outputHeading, outputDetailsContainer);
+
+	button.onclick = async () => {
+		outputHeading.innerText = '⏳ Loading...';
+		outputDetailsContent.innerText = '';
+		outputContainer.classList.remove('error');
+		outputContainer.classList.remove('empty');
+		outputContainer.classList.add('loading');
+		button.disabled = true;
+
+		try {
+			await onSubmit(itemLinkToId(input.value), outputHeading, outputDetailsContent);
+			outputContainer.classList.remove('loading');
+		} catch (error) {
+			outputHeading.innerText = `⚠️ Error: ${error}`;
+			outputContainer.classList.add('error');
+		}
+
+		button.disabled = false;
+	};
+
+	parent.appendChild(button);
+	parent.appendChild(outputContainer);
+};
+
+const checkForItemOnServer = async (
+	itemId: string, outputHeadingElement: HTMLElement, outputDetailsElement: HTMLElement,
+) => {
+	const fetchResult = await fetch(`/api/items/root:/${encodeURIComponent(itemId)}.md:/`);
+
+	if (fetchResult.ok) {
+		const result = await fetchResult.text();
+		outputHeadingElement.innerText = 'Item found!';
+		outputDetailsElement.innerText = result;
+	} else {
+		outputHeadingElement.innerText = `Item ${itemId}: ${fetchResult.statusText}`;
+		outputDetailsElement.innerText = '';
+	}
+};
+
+const checkForItemInInitialDiff = async (
+	itemId: string, outputHeadingElement: HTMLElement, outputDetailsElement: HTMLElement,
+) => {
+	let cursor: string|undefined = undefined;
+
+	const waitForTimeout = (timeout: number) => {
+		return new Promise<void>(resolve => {
+			setTimeout(() => resolve(), timeout);
+		});
+	};
+
+	const readDiff = async function*() {
+		let hasMore = true;
+		let page = 1;
+		while (hasMore) {
+			const fetchResult = await fetch(
+				`/api/items/root/delta${
+					cursor ? `?cursor=${encodeURIComponent(cursor)}` : ''
+				}`,
+			);
+			if (!fetchResult.ok) {
+				throw new Error(`Error fetching items: ${fetchResult.statusText}`);
+			}
+
+			const json = await fetchResult.json();
+			hasMore = json.has_more;
+			cursor = json.cursor;
+
+			for (const item of json.items) {
+				yield item;
+			}
+
+			outputHeadingElement.innerText = `Processing page ${page++}...`;
+
+			// Avoid sending requests too frequently
+			await waitForTimeout(200); // ms
+		}
+	};
+
+	const allItems = [];
+	const matches = [];
+	let stoppedEarly = false;
+	for await (const item of readDiff()) {
+		// Include console logging to provide more information if readDiff() fails.
+		// eslint-disable-next-line no-console
+		console.log('Checking item', item);
+
+		if (item.item_name === `${itemId}.md`) {
+			matches.push(item);
+			stoppedEarly = true;
+		}
+		allItems.push(item);
+	}
+
+	outputHeadingElement.innerText
+		= matches.length > 0 ? 'Found in initial sync diff' : `Item ${itemId}: Not in initial sync diff`;
+
+	const stoppedEarlyDescription = (
+		stoppedEarly ? '\n Stopped fetching items after finding a match. Item list is incomplete.' : ''
+	);
+	outputDetailsElement.innerText
+		= JSON.stringify(allItems, undefined, '  ') + stoppedEarlyDescription;
+};
+
+document.addEventListener('DOMContentLoaded', () => {
+	setUpItemChecker(document.querySelector('#note-on-server-check'), checkForItemOnServer);
+	setUpItemChecker(document.querySelector('#note-in-diff-check'), checkForItemInInitialDiff);
+});

packages/server/public/js/tsconfig.json

@@ -0,0 +1,15 @@
+{
+	"extends": "../../../../tsconfig.json",
+	"compilerOptions": {
+		// ES6 (when there are no imports/exports) generates code that can be run in a web browser.
+		"module": "ES6",
+		"declaration": false
+	},
+	"rootDir": ".",
+	"include": [
+		"**/*.ts"
+	],
+	"exclude": [
+		"*.test.ts"
+	]
+}

packages/server/src/routes/index/sync_debug.ts

@@ -0,0 +1,24 @@
+import { SubPath } from '../../utils/routeUtils';
+import Router from '../../utils/Router';
+import { RouteType } from '../../utils/types';
+import { AppContext } from '../../utils/types';
+import { ErrorMethodNotAllowed } from '../../utils/errors';
+import { contextSessionId } from '../../utils/requestUtils';
+import defaultView from '../../utils/defaultView';
+
+const router = new Router(RouteType.Web);
+
+router.get('sync_debug', async (_path: SubPath, ctx: AppContext) => {
+	contextSessionId(ctx);
+
+	if (ctx.method !== 'GET') {
+		throw new ErrorMethodNotAllowed();
+	}
+
+	const view = defaultView('sync_debug', 'Sync Debug');
+	view.cssFiles = ['index/sync_debug'];
+	view.jsFiles = ['index/sync_debug'];
+	return view;
+});
+
+export default router;

packages/server/src/routes/routes.ts

@@ -19,6 +19,7 @@ import adminUserDeletions from './admin/user_deletions';
 import adminUsers from './admin/users';
 
 import indexChanges from './index/changes';
+import indexSyncDebug from './index/sync_debug';
 import indexHelp from './index/help';
 import indexHome from './index/home';
 import indexItems from './index/items';
@@ -56,6 +57,7 @@ const routes: Routers = {
 	'admin/users': adminUsers,
 
 	'changes': indexChanges,
+	'sync_debug': indexSyncDebug,
 	'help': indexHelp,
 	'home': indexHome,
 	'items': indexItems,

packages/server/src/views/index/sync_debug.mustache

@@ -0,0 +1,24 @@
+<h1 class="title">Debugging</h1>
+<p>
+	Use these tools to debug sync issues with Joplin Server.
+</p>
+
+<div id="debug-tools-container">
+	<fieldset id="note-on-server-check">
+		<legend>
+			Check if an item (note, notebook, resource, tag) is present on the server
+		</legend>
+		<label for="note-on-server-input">External link or ID:</label>
+		<input type="text" id="note-on-server-input"/>
+		<noscript>JavaScript required</noscript>
+	</fieldset>
+
+	<fieldset id="note-in-diff-check">
+		<legend>
+			Check if an item is sent to Joplin to download on an initial sync
+		</legend>
+		<label for="note-on-server-input">External link or ID:</label>
+		<input type="text" id="note-in-diff-input"/>
+		<noscript>JavaScript required</noscript>
+	</fieldset>
+</div>

packages/server/tsconfig.json

@@ -10,5 +10,6 @@
 	],
 	"exclude": [
 		"**/node_modules",
-	],
+		"public/js/"
+	]
 }

packages/tools/gulp/tasks/updateIgnoredTypeScriptBuild.js

@@ -33,7 +33,8 @@ module.exports = {
 				'packages/app-mobile/ios/**',
 				'packages/fork-sax/**',
 				'packages/lib/plugin_types/**',
-				'packages/server/**',
+				'packages/server/src/**',
+				'packages/server/dist/**',
 				'packages/utils/**',
 			],
 		}).filter(f => !f.endsWith('.d.ts'));

packages/utils/package.json

@@ -20,6 +20,7 @@
   "scripts": {
     "tsc": "tsc --project tsconfig.json",
     "watch": "tsc --watch --preserveWatchOutput --project tsconfig.json",
+    "build": "yarn run tsc",
     "test": "jest --verbose=false",
     "test-ci": "yarn test"
   },