From 32b3cd388700de0e68b3e2f4f4bfd17f01832eae Mon Sep 17 00:00:00 2001 From: Shaun Hamilton Date: Mon, 25 Mar 2024 10:26:00 +0200 Subject: [PATCH 1/4] feat: use script command for terminal logs --- .freeCodeCamp/tooling/test-utils.js | 16 ++++++++ cli/src/fs.rs | 2 +- docs/src/CHANGELOG.md | 11 +++++- docs/src/SUMMARY.md | 1 - docs/src/cli.md | 2 +- docs/src/project-syntax.md | 16 +++----- docs/src/roadmap.md | 10 ----- docs/src/testing/globals.md | 2 +- docs/src/testing/test-utilities.md | 44 ++++++++++++++++++++- package.json | 2 +- self/bash/.bashrc | 13 +++--- self/config/projects.json | 16 +++++++- self/curriculum/locales/english/dot-logs.md | 34 ++++++++++++++++ self/dot-logs/.gitkeep | 0 14 files changed, 134 insertions(+), 35 deletions(-) delete mode 100644 docs/src/roadmap.md create mode 100644 self/curriculum/locales/english/dot-logs.md create mode 100644 self/dot-logs/.gitkeep diff --git a/.freeCodeCamp/tooling/test-utils.js b/.freeCodeCamp/tooling/test-utils.js index 146602de..f1bb4e42 100644 --- a/.freeCodeCamp/tooling/test-utils.js +++ b/.freeCodeCamp/tooling/test-utils.js @@ -12,6 +12,7 @@ export const PATH_TERMINAL_OUT = join(ROOT, '.logs/.terminal_out.log'); export const PATH_BASH_HISTORY = join(ROOT, '.logs/.bash_history.log'); export const PATH_CWD = join(ROOT, '.logs/.cwd.log'); export const PATH_TEMP = join(ROOT, '.logs/.temp.log'); +export const PATH_SCRIPT_OUT = join(ROOT, '.logs/.script_out.log'); /** * @typedef ControlWrapperOptions @@ -110,9 +111,22 @@ async function getLastCWD(howManyBack = 0) { return lastLog; } +/** + * Get the `.logs/.script_out.log` file contents, or `throw` if not found + * @returns {Promise} The `.script_out.log` file contents + */ +async function getScriptOut() { + const scriptLogs = await readFile(PATH_SCRIPT_OUT, { + encoding: 'utf8', + flag: 'a+' + }); + return scriptLogs; +} + /** * Get the `.logs/.temp.log` file contents, or `throw` if not found * @returns {Promise} The `.temp.log` file contents + * @deprecated Use `getScriptOut` instead */ async function getTemp() { const tempLogs = await readFile(PATH_TEMP, { @@ -125,6 +139,7 @@ async function getTemp() { /** * Get the `.logs/.terminal_out.log` file contents, or `throw` if not found * @returns {Promise} The `.terminal_out.log` file contents + * @deprecated Use `getScriptOut` instead */ async function getTerminalOutput() { const terminalLogs = await readFile(PATH_TERMINAL_OUT, { @@ -151,6 +166,7 @@ const __helpers = { getCWD, getLastCommand, getLastCWD, + getScriptOut, getTemp, getTerminalOutput, importSansCache diff --git a/cli/src/fs.rs b/cli/src/fs.rs index d8f6ef9a..e2ca8d0f 100644 --- a/cli/src/fs.rs +++ b/cli/src/fs.rs @@ -191,7 +191,7 @@ impl Course { }, hot_reload: Some(HotReload { ignore: vec![ - ".logs/.temp.log".to_string(), + ".logs/.script_out.log".to_string(), "config/".to_string(), "/node_modules/".to_string(), ".git".to_string(), diff --git a/docs/src/CHANGELOG.md b/docs/src/CHANGELOG.md index ea4574f7..a63cf1d8 100644 --- a/docs/src/CHANGELOG.md +++ b/docs/src/CHANGELOG.md @@ -1,12 +1,19 @@ # Changelog -## [3.6.0] - +## [4.0.0] + +### Change + +- Remove `__helpers.getTemp` +- Remove `__helpers.getTerminalOut` + +## [3.6.0] - 2024-03-20 ### Add - Use bash's `script` command to record terminal input and output - - Gated behind a feature flag - Existing `.logs/` files will be deprecated in favour of `script` command in `4.0` + - `__helpers.getScriptOut` to get `.logs/.script_out.log` file ## [3.5.1] - 2024-03-19 diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 36789063..0b02052b 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -22,4 +22,3 @@ - [Client Injection](./client-injection.md) - [Contributing](./contributing.md) - [CHANGELOG](./CHANGELOG.md) -- [Roadmap](./roadmap.md) diff --git a/docs/src/cli.md b/docs/src/cli.md index 7a47cdf8..8832fc13 100644 --- a/docs/src/cli.md +++ b/docs/src/cli.md @@ -30,4 +30,4 @@ To add a project to an existing course: create-freecodecamp-os-app add-project ``` -The version of the CLI is tied to the version of `freecodecamp-os`. Some options may not be available if the version of the CLI is not compatible with the version of `freecodecamp-os` that is installed. +The major version of the CLI is tied to the version of `freecodecamp-os`. Some options may not be available if the version of the CLI is not compatible with the version of `freecodecamp-os` that is installed. diff --git a/docs/src/project-syntax.md b/docs/src/project-syntax.md index 4e49ea36..239312fc 100644 --- a/docs/src/project-syntax.md +++ b/docs/src/project-syntax.md @@ -32,20 +32,15 @@ This is a description. ## ``` +The first `json` code block is used for `meta`. + ```admonish note title="" Zero-based numbering, because of course ``` -````admonish example collapsible=true -```markdown -## 0 -``` -```` - -### `meta` - +`````admonish example collapsible=true ````markdown -## +## 0 ```json { @@ -54,8 +49,9 @@ Zero-based numbering, because of course } ``` ```` +````` -The `meta.watch` field is used to specify specific files to watch during a lesson. The `meta.ignore` field is used to specify specific files to ignore during a lesson. The watcher is affected once on lesson load. +The `watch` field is used to specify specific files to watch during a lesson. The `ignore` field is used to specify specific files to ignore during a lesson. The watcher is affected once on lesson load. ```admonish note title="" The `watch` and `ignore` fields are optional. It does not make sense to provide both at the same time. diff --git a/docs/src/roadmap.md b/docs/src/roadmap.md deleted file mode 100644 index 5b12b98f..00000000 --- a/docs/src/roadmap.md +++ /dev/null @@ -1,10 +0,0 @@ -# Roadmap - -For the most part, this roadmap outlines todos for `freecodecamp-os`. If this roadmap is empty, then there are no todos 🎉 - -## Documentation - -## Features - -- [ ] Loader to show progress of "Reset Step" -- [ ] Crowdin translation integration diff --git a/docs/src/testing/globals.md b/docs/src/testing/globals.md index 8ad85436..ca17c06f 100644 --- a/docs/src/testing/globals.md +++ b/docs/src/testing/globals.md @@ -33,7 +33,7 @@ The root of the workspace. ### `watcher` ```admonish note -This is only available in the `beforeAll` and `beforeEach` context - on the main thread. +This is only available in the `beforeAll` and `afterAll` context - on the main thread. ``` The [Chokidar](https://www.npmjs.com/package/chokidar) `FSWatcher` instance. diff --git a/docs/src/testing/test-utilities.md b/docs/src/testing/test-utilities.md index c9bb42be..e5f0d16f 100644 --- a/docs/src/testing/test-utilities.md +++ b/docs/src/testing/test-utilities.md @@ -1,9 +1,20 @@ -# Test Utilities +# Test Utilities The test utilities are exported/global objects available in the test runner. These are referred to as _"helpers"_, and the included helpers are exported from [https://github.com/freeCodeCamp/freeCodeCampOS/blob/main/.freeCodeCamp/tooling/test-utils.js](https://github.com/freeCodeCamp/freeCodeCampOS/blob/main/.freeCodeCamp/tooling/test-utils.js). Many of the exported functions are _convinience wrappers_ around Nodejs' `fs` and `child_process` modules. Specifically, they make use of the global `ROOT` variable to run the functions relative to the root of the workspace. +- [`controlWrapper`](#controlwrapper) +- [`getBashHistory`](#getbashhistory) +- [`getCommandOutput`](#getcommandoutput) +- [`getCWD`](#getcwd) +- [`getLastCommand`](#getlastcommand) +- [`getLastCWD`](#getlastcwd) +- [`getScriptOut`](#getscriptout) +- [`getTemp`](#gettemp) +- [`getTerminalOutput`](#getterminaloutput) +- [`importSansCache`](#importsanscache) + ## `controlWrapper` Wraps a function in an interval to retry until it does not throw or times out. @@ -110,8 +121,34 @@ function getLastCWD(n = 0): Promise; const lastCWD = await __helpers.getLastCWD(); ``` +## `getScriptOut` + +Get the `.logs/.script_out.log` file contents. + +```admonish danger title="Safety" +Throws if file does not exist, or if read permission is denied. +``` + +```typescript +function getScriptOut(): Promise; +``` + +```javascript +const scriptOut = await __helpers.getScriptOut(); +``` + +```admonish note +Use the output of the `.script_out.log` file at your own risk. This file is raw input from the terminal including ANSI escape codes. +``` + +TODO: use `--log-in` to "watch" `--log-out`. When `--log-in` changes, read from `--log-out`. + ## `getTemp` +```admonish warning title="3.6.0" +Deprecated in favour of [`getScriptOut`](#getscriptout). +``` + Get the `.logs/.temp.log` file contents. ```admonish danger title="Safety" @@ -134,6 +171,11 @@ Output varies depending on emulator, terminal size, order text is typed, etc. Fo ## `getTerminalOutput` +```admonish warning title="3.6.0" +Deprecated in favour of [`getScriptOut`](#getscriptout) + +``` + Get the `.logs/.terminal_out.log` file contents. ```admonish danger title="Safety" diff --git a/package.json b/package.json index 9df150dd..3b918719 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@freecodecamp/freecodecamp-os", "author": "freeCodeCamp", - "version": "3.5.1", + "version": "3.6.0", "description": "Package used for freeCodeCamp projects with the freeCodeCamp Courses VSCode extension", "scripts": { "build:client": "NODE_ENV=production webpack --config ./.freeCodeCamp/webpack.config.cjs", diff --git a/self/bash/.bashrc b/self/bash/.bashrc index 1dfbac61..de532ed3 100644 --- a/self/bash/.bashrc +++ b/self/bash/.bashrc @@ -118,15 +118,16 @@ fi PS1='\[\]\u\[\] \[\]\w\[\]$(__git_ps1 " (%s)") $ ' -for i in $(ls -A $HOME/.bashrc.d/); do source $HOME/.bashrc.d/$i; done - - # freeCodeCamp - Needed for most tests to work WD=/workspace/freeCodeCampOS # Ensure `$WD/.logs/` directory and files exist mkdir -p $WD/.logs/ -touch $WD/.logs/.bash_history.log $WD/.logs/.cwd.log $WD/.logs/.history_cwd.log $WD/.logs/.terminal_out.log $WD/.logs/.temp.log +touch $WD/.logs/.bash_history.log $WD/.logs/.cwd.log $WD/.logs/.history_cwd.log -PROMPT_COMMAND='>| $WD/.logs/.terminal_out.log && cat $WD/.logs/.temp.log >| $WD/.logs/.terminal_out.log && truncate -s 0 $WD/.logs/.temp.log; echo $PWD >> $WD/.logs/.cwd.log; history -a $WD/.logs/.bash_history.log; echo $PWD\$ $(history | tail -n 1) >> $WD/.logs/.history_cwd.log;' -exec > >(tee -ia $WD/.logs/.temp.log) 2>&1 +PROMPT_COMMAND='echo $PWD >> $WD/.logs/.cwd.log; history -a $WD/.logs/.bash_history.log; echo $PWD\$ $(history | tail -n 1) >> $WD/.logs/.history_cwd.log;' + +if test -t 0 ; then + script --flush --append --log-out $WD/.logs/.script_out.log + exit +fi diff --git a/self/config/projects.json b/self/config/projects.json index 94ab5d6a..92359931 100644 --- a/self/config/projects.json +++ b/self/config/projects.json @@ -68,5 +68,19 @@ "blockingTests": false, "breakOnFailure": false, "numberOfLessons": 3 + }, + { + "id": 5, + "dashedName": "dot-logs", + "isIntegrated": false, + "isPublic": true, + "currentLesson": 0, + "runTestsOnWatch": true, + "seedEveryLesson": false, + "isResetEnabled": false, + "numberofLessons": null, + "blockingTests": false, + "breakOnFailure": false, + "numberOfLessons": 2 } -] +] \ No newline at end of file diff --git a/self/curriculum/locales/english/dot-logs.md b/self/curriculum/locales/english/dot-logs.md new file mode 100644 index 00000000..a0bc26d4 --- /dev/null +++ b/self/curriculum/locales/english/dot-logs.md @@ -0,0 +1,34 @@ +# Dot Logs + +The `.logs/` folder and features + +## 0 + +### --description-- + +In a new terminal, start typing `dot logs`, and the tests should run. + +### --tests-- + +Placeholder test. + +```js +const scriptOut = await __helpers.getScriptOut(); +assert.match(scriptOut, /dot logs/); +``` + +## 1 + +### --description-- + +As + +### --tests-- + +fail + +```js +assert.fail(); +``` + +## --fcc-end-- diff --git a/self/dot-logs/.gitkeep b/self/dot-logs/.gitkeep new file mode 100644 index 00000000..e69de29b From f2add51409229d203184be1893823fdcbaa7d932 Mon Sep 17 00:00:00 2001 From: Shaun Hamilton Date: Mon, 8 Apr 2024 16:16:48 +0200 Subject: [PATCH 2/4] add watcher tooling --- .freeCodeCamp/tests/watcher.test.js | 38 ++++++++++ .freeCodeCamp/tooling/watcher/watcher.js | 91 ++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 .freeCodeCamp/tests/watcher.test.js create mode 100644 .freeCodeCamp/tooling/watcher/watcher.js diff --git a/.freeCodeCamp/tests/watcher.test.js b/.freeCodeCamp/tests/watcher.test.js new file mode 100644 index 00000000..36c1de6b --- /dev/null +++ b/.freeCodeCamp/tests/watcher.test.js @@ -0,0 +1,38 @@ +import { it, describe, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { + compareEquivalency, + maybeAddRelativePathToPTI +} from '../tooling/watcher/watcher.js'; +import { PATHS_TO_IGNORE } from '../tooling/hot-reload.js'; + +describe('watcher', async () => { + let initial_PTI; + before(() => { + initial_PTI = [...PATHS_TO_IGNORE]; + const TEST_PTI = ['/node_modules/']; + PATHS_TO_IGNORE.splice(0, PATHS_TO_IGNORE.length, ...TEST_PTI); + }); + after(() => { + PATHS_TO_IGNORE.splice(0, PATHS_TO_IGNORE.length, ...initial_PTI); + }); + + it('compareEquivalency', () => { + const pti = ['/a', 'b', '/c/']; + const equivalent_paths = ['/a/1', '/a/b', '/c/1/2']; + const unequivalent_paths = ['a/1', 'c', '/d']; + + pti.forEach((p, i) => { + const ep = equivalent_paths[i]; + assert(compareEquivalency(p, ep), `Expected ${p} ~= ${ep}`); + const up = unequivalent_paths[i]; + assert(!compareEquivalency(p, up), `Expected ${p} !~= ${up}`); + }); + }); + + it('maybeAddRelativePathToPTI', () => { + assert.deepEqual(PATHS_TO_IGNORE, ['/node_modules/']); + maybeAddRelativePathToPTI('node_modules'); + assert.deepEqual(PATHS_TO_IGNORE, ['node_modules']); + }); +}); diff --git a/.freeCodeCamp/tooling/watcher/watcher.js b/.freeCodeCamp/tooling/watcher/watcher.js new file mode 100644 index 00000000..60d65146 --- /dev/null +++ b/.freeCodeCamp/tooling/watcher/watcher.js @@ -0,0 +1,91 @@ +import path from 'path'; +import { PATHS_TO_IGNORE } from '../hot-reload.js'; +import { ROOT } from '../env.js'; + +// TODO: +// 1. Ensure all paths start with `ROOT` so comparisions work +// 2. Allow directory paths to optionally end with `/` +// 3. If relative path does not start with `/`, do not prepend `ROOT` +// - This indicates the path can be anywhere in the tree + +// NOTES: +// - `name` is always the full path including `ROOT` +// - `PATHS_TO_IGNORE` should be relative to `ROOT` +// - Must include the full path to the file/directory wanting to be ignored from `ROOT` + +export function unwatchPath(pathRelativeToRoot) {} + +export function watchPath(pathRelativeToRoot) {} + +/** + * Add `pathRelativeToRoot` to `PATHS_TO_IGNORE`, if it is not already **equivalently** added + * + * **Examples** + * + * ```js + * const PATHS_TO_IGNORE = ["node_modules/"]; + * const pathRelativeToRoot = "/node_modules/foo/bar"; + * maybeAddRelativePathToPTI(pathRelativeToRoot); + * console.log(PATHS_TO_IGNORE); + * // ["node_modules/"] + * ``` + * `PATHS_TO_IGNORE` is still `["node_modules/"]`, because `/node_modules/foo/bar` is equivalently added. + * + * --- + * + * ```js + * const PATHS_TO_IGNORE = ["/node_modules/"]; + * const pathRelativeToRoot = "node_modules/foo/bar"; + * maybeAddRelativePathToPTI(pathRelativeToRoot); + * console.log(PATHS_TO_IGNORE); + * // ["/node_modules/", "node_modules/foo/bar"] + * ``` + * + * `node_modules/foo/bar` is added, because `/node_modules/` does not encompass all equivalents. + * + * @param {string} pathRelativeToRoot + */ +export function maybeAddRelativePathToPTI(pathRelativeToRoot) { + if (!PATHS_TO_IGNORE.some(p => compareEquivalency(p, pathRelativeToRoot))) { + // If reverse equivalency exists for any elements already in PATHS_TO_IGNORE, remove less covered path + PATHS_TO_IGNORE.forEach(p => { + if (compareEquivalency(pathRelativeToRoot, p)) { + PATHS_TO_IGNORE.splice(PATHS_TO_IGNORE.indexOf(p), 1); + } + }); + + PATHS_TO_IGNORE.push(pathRelativeToRoot); + } +} + +/** + * Compares two paths for equivalency: + * - If path starts with `/`, prepends `ROOT`, and compares full path equality + * - Else, compares path as `/path/` string + * - Allows optional trailing `/` + * - If one path is a subset of the other, then they are equivalent + * @param {string} covered_path Path already covered + * @param {string} compated_path Path to be covered + * @returns {boolean} If `covered_path` accounts for `compared_path` + */ +export function compareEquivalency(covered_path, compared_path) { + const is_absolute_covered_path = path.isAbsolute(covered_path); + const is_absolute_compared_path = path.isAbsolute(compared_path); + + let path_covered = is_absolute_covered_path + ? path.join(ROOT, covered_path) + : covered_path; + let path_compared = is_absolute_compared_path + ? path.join(ROOT, compared_path) + : compared_path; + + // Append trailing `/` if not present + if (!path_covered.endsWith('/')) { + path_covered += '/'; + } + if (!path_compared.endsWith('/')) { + path_compared += '/'; + } + + return path_compared.includes(path_covered); +} From 2deecc5c4f2fbc6f2879828a09bbefa363f3f7e9 Mon Sep 17 00:00:00 2001 From: Shaun Hamilton Date: Fri, 12 Apr 2024 17:49:12 +0200 Subject: [PATCH 3/4] fix: add watcher utils --- .freeCodeCamp/tests/watcher.test.js | 48 ++++++-- .freeCodeCamp/tooling/hot-reload.js | 103 ++--------------- .freeCodeCamp/tooling/lesson.js | 42 +++---- .freeCodeCamp/tooling/seed.js | 14 +-- .freeCodeCamp/tooling/server.js | 2 +- .freeCodeCamp/tooling/test-utils.js | 26 +++++ .freeCodeCamp/tooling/tests/main.js | 11 +- .freeCodeCamp/tooling/watcher/watcher.js | 116 +++++++++++++++++--- docs/src/testing/test-utilities.md | 21 ++++ package-lock.json | 4 +- package.json | 1 - self/curriculum/locales/english/dot-logs.md | 10 +- self/package.json | 7 +- 13 files changed, 244 insertions(+), 161 deletions(-) diff --git a/.freeCodeCamp/tests/watcher.test.js b/.freeCodeCamp/tests/watcher.test.js index 36c1de6b..405151ba 100644 --- a/.freeCodeCamp/tests/watcher.test.js +++ b/.freeCodeCamp/tests/watcher.test.js @@ -1,21 +1,31 @@ -import { it, describe, before, after } from 'node:test'; +import { it, describe, before, beforeEach, after, afterEach } from 'node:test'; import assert from 'node:assert/strict'; import { compareEquivalency, - maybeAddRelativePathToPTI + maybeAddRelativePath, + shouldWatch, + PATHS_TO_WATCH, + PATHS_TO_IGNORE, + unwatchPath, + watchPath } from '../tooling/watcher/watcher.js'; -import { PATHS_TO_IGNORE } from '../tooling/hot-reload.js'; + +import { watcher } from '../tooling/hot-reload.js'; describe('watcher', async () => { let initial_PTI; before(() => { initial_PTI = [...PATHS_TO_IGNORE]; - const TEST_PTI = ['/node_modules/']; - PATHS_TO_IGNORE.splice(0, PATHS_TO_IGNORE.length, ...TEST_PTI); }); - after(() => { + beforeEach(() => { + PATHS_TO_IGNORE.splice(0, PATHS_TO_IGNORE.length); + }); + afterEach(() => { PATHS_TO_IGNORE.splice(0, PATHS_TO_IGNORE.length, ...initial_PTI); }); + after(() => { + watcher.close(); + }); it('compareEquivalency', () => { const pti = ['/a', 'b', '/c/']; @@ -30,9 +40,31 @@ describe('watcher', async () => { }); }); - it('maybeAddRelativePathToPTI', () => { + it('maybeAddRelativePath', () => { + assert.deepEqual(PATHS_TO_IGNORE, []); + maybeAddRelativePath(PATHS_TO_IGNORE, '/node_modules/'); assert.deepEqual(PATHS_TO_IGNORE, ['/node_modules/']); - maybeAddRelativePathToPTI('node_modules'); + maybeAddRelativePath(PATHS_TO_IGNORE, 'node_modules'); + assert.deepEqual(PATHS_TO_IGNORE, ['node_modules']); + maybeAddRelativePath(PATHS_TO_IGNORE, '/node_modules/foo/bar'); assert.deepEqual(PATHS_TO_IGNORE, ['node_modules']); + maybeAddRelativePath(PATHS_TO_IGNORE, 'a/node_modules'); + assert.deepEqual(PATHS_TO_IGNORE, ['node_modules']); + maybeAddRelativePath(PATHS_TO_IGNORE, '/a/node_modules/b/'); + assert.deepEqual(PATHS_TO_IGNORE, ['node_modules']); + + maybeAddRelativePath(PATHS_TO_IGNORE, '/a/b/'); + assert.deepEqual(PATHS_TO_IGNORE, ['node_modules', '/a/b/']); + }); + + it('shouldWatch', () => { + assert(shouldWatch('node_modules/foo/bar')); + unwatchPath('/node_modules/'); + assert(!shouldWatch('/node_modules/foo/bar')); + assert(shouldWatch('example/node_modules/foo/bar')); + unwatchPath('node_modules'); + assert(!shouldWatch('example/node_modules/foo/bar')); + watchPath('/node_modules/foo'); + assert(shouldWatch('/node_modules/foo/bar')); }); }); diff --git a/.freeCodeCamp/tooling/hot-reload.js b/.freeCodeCamp/tooling/hot-reload.js index b0f2a40e..5e8d4cde 100644 --- a/.freeCodeCamp/tooling/hot-reload.js +++ b/.freeCodeCamp/tooling/hot-reload.js @@ -1,31 +1,20 @@ -// This file handles the watching of the /curriculum folder for changes -// and executing the command to run the tests for the next (current) lesson -import { getState, getProjectConfig, ROOT, freeCodeCampConfig } from './env.js'; +import { getState, getProjectConfig, ROOT } from './env.js'; import { runLesson } from './lesson.js'; import { runTests } from './tests/main.js'; +// `watch` from `node:fs` was investigated, but it fired too many events multiple times +// as of node^21 import { watch } from 'chokidar'; import { logover } from './logger.js'; -import path from 'path'; -import { readdir } from 'fs/promises'; -const defaultPathsToIgnore = [ - '.logs/.temp.log', - 'config/', - '/node_modules/', - '.git/', - '/target/', - '/test-ledger/' -]; - -export const pathsToIgnore = - freeCodeCampConfig.hotReload?.ignore || defaultPathsToIgnore; +import { shouldWatch } from './watcher/watcher.js'; +import { join } from 'path'; export const watcher = watch(ROOT, { ignoreInitial: true, - ignored: path => pathsToIgnore.some(p => path.includes(p)) + ignored: join(ROOT, 'node_modules/@freecodecamp/freecodecamp-os') }); -export function hotReload(ws, pathsToIgnore = defaultPathsToIgnore) { +export function hotReload(ws) { logover.info(`Watching for file changes on ${ROOT}`); let isWait = false; let testsRunning = false; @@ -37,7 +26,10 @@ export function hotReload(ws, pathsToIgnore = defaultPathsToIgnore) { watcher.removeAllListeners('all'); watcher.on('all', async (event, name) => { - if (name && !pathsToIgnore.find(p => name.includes(p))) { + // If path in `PATHS_TO_WATCH`, then watch - ignore `PATHS_TO_IGNORE` + // Else if path in `PATHS_TO_IGNORE`, then ignore + // Else, watch + if (name && shouldWatch(name)) { if (isWait) return; const { currentProject } = await getState(); if (!currentProject) { @@ -65,76 +57,3 @@ export function hotReload(ws, pathsToIgnore = defaultPathsToIgnore) { } }); } - -/** - * Stops the global `watcher` from watching the entire workspace. - */ -export function unwatchAll() { - const watched = watcher.getWatched(); - for (const [dir, files] of Object.entries(watched)) { - for (const file of files) { - watcher.unwatch(path.join(dir, file)); - } - } -} - -// Need to handle -// From ROOT, must add all directories before file/s -// path.dirname... all the way to ROOT -// path.isAbsolute to find out if what was passed into `meta` is absolute or relative -// path.parse to get the dir and base -// path.relative(ROOT, path) to get the relative path from ROOT -// path.resolve directly on `meta`? -/** - * **Example:** - * - Assuming ROOT is `/home/freeCodeCampOS/self` - * - Takes `lesson-watcher/src/watched.js` - * - Calls `watcher.add` on each of these in order: - * - `/home/freeCodeCampOS/self` - * - `/home/freeCodeCampOS/self/lesson-watcher` - * - `/home/freeCodeCampOS/self/lesson-watcher/src` - * - `/home/freeCodeCampOS/self/lesson-watcher/src/watched.js` - * @param {string} pathRelativeToRoot - */ -export function watchPathRelativeToRoot(pathRelativeToRoot) { - const paths = getAllPathsWithRoot(pathRelativeToRoot); - for (const path of paths) { - watcher.add(path); - } -} - -function getAllPathsWithRoot(pathRelativeToRoot) { - const paths = []; - let currentPath = pathRelativeToRoot; - while (currentPath !== ROOT) { - paths.push(currentPath); - currentPath = path.dirname(currentPath); - } - paths.push(ROOT); - // The order does not _seem_ to matter, but the theory says it should - return paths.reverse(); -} - -/** - * Adds all folders and files to the `watcher` instance. - * - * Does nothing with the `pathsToIgnore`, because they are already ignored by the `watcher`. - */ -export async function watchAll() { - await watchPath(ROOT); -} - -async function watchPath(rootPath) { - const paths = await readdir(rootPath, { withFileTypes: true }); - for (const p of paths) { - const fullPath = path.join(rootPath, p.name); - // if (pathsToIgnore.find(i => fullPath.includes(i))) { - // console.log('Ignoring: ', fullPath); - // continue; - // } - watcher.add(fullPath); - if (p.isDirectory()) { - await watchPath(fullPath); - } - } -} diff --git a/.freeCodeCamp/tooling/lesson.js b/.freeCodeCamp/tooling/lesson.js index 29ee50d8..9bff2640 100644 --- a/.freeCodeCamp/tooling/lesson.js +++ b/.freeCodeCamp/tooling/lesson.js @@ -1,5 +1,3 @@ -// This file parses answer files for lesson content -import { join } from 'path'; import { updateDescription, updateProjectHeading, @@ -8,16 +6,11 @@ import { updateError, resetBottomPanel } from './client-socks.js'; -import { ROOT, getState, getProjectConfig, setState } from './env.js'; +import { getState, getProjectConfig } from './env.js'; import { logover } from './logger.js'; import { seedLesson } from './seed.js'; import { pluginEvents } from '../plugin/index.js'; -import { - unwatchAll, - watchAll, - watchPathRelativeToRoot, - watcher -} from './hot-reload.js'; +import { watchPath, unwatchPath, resetPathLists } from './watcher/watcher.js'; /** * Runs the lesson from the `projectDashedName` config. @@ -27,14 +20,14 @@ import { export async function runLesson(ws, projectDashedName) { const project = await getProjectConfig(projectDashedName); const { isIntegrated, dashedName, seedEveryLesson, currentLesson } = project; - const { lastSeed, lastWatchChange } = await getState(); + const { lastSeed } = await getState(); try { const { description, seed, isForce, tests, meta } = await pluginEvents.getLesson(projectDashedName, currentLesson); // TODO: Consider performance optimizations // - Do not run at all if whole project does not contain any `meta`. - await handleWatcher(meta, { lastWatchChange, currentLesson }); + await handleWatcher(meta); if (currentLesson === 0) { await pluginEvents.onProjectStart(project); @@ -83,22 +76,17 @@ export async function runLesson(ws, projectDashedName) { } } -async function handleWatcher(meta, { lastWatchChange, currentLesson }) { - // Calling `watcher` methods takes a performance hit. So, check is behind a check that the lesson has changed. - if (lastWatchChange !== currentLesson) { - if (meta?.watch) { - unwatchAll(); - for (const path of meta.watch) { - const toWatch = join(ROOT, path); - watchPathRelativeToRoot(toWatch); - } - } else if (meta?.ignore) { - await watchAll(); - watcher.unwatch(meta.ignore); - } else { - // Reset watcher back to default/freecodecamp.conf.json - await watchAll(); +async function handleWatcher(meta) { + if (meta?.watch) { + for (const path of meta.watch) { + watchPath(path); + } + } else if (meta?.ignore) { + for (const path of meta.ignore) { + unwatchPath(path); } + } else { + // Reset watcher back to default/freecodecamp.conf.json + resetPathLists(); } - await setState({ lastWatchChange: currentLesson }); } diff --git a/.freeCodeCamp/tooling/seed.js b/.freeCodeCamp/tooling/seed.js index 47851c26..fb77d1e4 100644 --- a/.freeCodeCamp/tooling/seed.js +++ b/.freeCodeCamp/tooling/seed.js @@ -1,19 +1,13 @@ // This file handles seeding the lesson contents with the seed in markdown. import { join } from 'path'; -import { - ROOT, - getState, - freeCodeCampConfig, - getProjectConfig, - setState -} from './env.js'; +import { ROOT, getProjectConfig, setState } from './env.js'; import { writeFile } from 'fs/promises'; import { promisify } from 'util'; import { exec } from 'child_process'; import { logover } from './logger.js'; import { updateLoader, updateError } from './client-socks.js'; -import { watcher } from './hot-reload.js'; import { pluginEvents } from '../plugin/index.js'; +import { resetPathLists, unwatchPath } from './watcher/watcher.js'; const execute = promisify(exec); /** @@ -104,9 +98,9 @@ export async function runLessonSeed(seed, currentLesson) { } else { const { filePath, fileSeed } = cmdOrFile; // Stop watching file being seeded to prevent triggering tests on hot reload - watcher.unwatch(filePath); + unwatchPath(filePath); await runSeed(fileSeed, filePath); - watcher.add(filePath); + resetPathLists(); } } } catch (e) { diff --git a/.freeCodeCamp/tooling/server.js b/.freeCodeCamp/tooling/server.js index 43da20f4..21f34116 100644 --- a/.freeCodeCamp/tooling/server.js +++ b/.freeCodeCamp/tooling/server.js @@ -234,7 +234,7 @@ const handle = { const wss = new WebSocketServer({ server }); wss.on('connection', function connection(ws) { - hotReload(ws, freeCodeCampConfig.hotReload?.ignore); + hotReload(ws); ws.on('message', function message(data) { const parsedData = parseBuffer(data); handle[parsedData.event]?.(ws, parsedData); diff --git a/.freeCodeCamp/tooling/test-utils.js b/.freeCodeCamp/tooling/test-utils.js index f1bb4e42..29a410c0 100644 --- a/.freeCodeCamp/tooling/test-utils.js +++ b/.freeCodeCamp/tooling/test-utils.js @@ -13,6 +13,7 @@ export const PATH_BASH_HISTORY = join(ROOT, '.logs/.bash_history.log'); export const PATH_CWD = join(ROOT, '.logs/.cwd.log'); export const PATH_TEMP = join(ROOT, '.logs/.temp.log'); export const PATH_SCRIPT_OUT = join(ROOT, '.logs/.script_out.log'); +export const PATH_SCRIPT_IN = join(ROOT, '.logs/.script_in.log'); /** * @typedef ControlWrapperOptions @@ -111,6 +112,29 @@ async function getLastCWD(howManyBack = 0) { return lastLog; } +/** + * Get the `.logs/.script_in.log` file contents, or `throw` if not found + * @returns {Promise} The `.script_in.log` file contents + */ +async function getScriptIn() { + const scriptLogs = await readFile(PATH_SCRIPT_IN, { + encoding: 'utf-8', + flag: 'a+' + }); + return scriptLogs; +} + +async function getScriptInEquivalent() { + const scriptIn = await getScriptIn(); + // TODO: Decide if removing the `^C` is necessary + let scriptInEquivalent = scriptIn.replace('\u0003', ''); + while (scriptInEquivalent.indexOf('\u007f') !== -1) { + scriptInEquivalent = scriptInEquivalent.replace(/.?\u007f/s, ''); + } + + return scriptInEquivalent; +} + /** * Get the `.logs/.script_out.log` file contents, or `throw` if not found * @returns {Promise} The `.script_out.log` file contents @@ -166,6 +190,8 @@ const __helpers = { getCWD, getLastCommand, getLastCWD, + getScriptIn, + getScriptInEquivalent, getScriptOut, getTemp, getTerminalOutput, diff --git a/.freeCodeCamp/tooling/tests/main.js b/.freeCodeCamp/tooling/tests/main.js index db7c135c..dfb7a66c 100644 --- a/.freeCodeCamp/tooling/tests/main.js +++ b/.freeCodeCamp/tooling/tests/main.js @@ -1,9 +1,13 @@ import { assert, AssertionError, expect, config as chaiConfig } from 'chai'; -import { watcher } from '../hot-reload.js'; import { logover } from '../logger.js'; -import { getProjectConfig, getState, setProjectConfig } from '../env.js'; -import { freeCodeCampConfig, ROOT } from '../env.js'; +import { + getProjectConfig, + getState, + setProjectConfig, + freeCodeCampConfig, + ROOT +} from '../env.js'; import { updateTest, updateTests, @@ -17,6 +21,7 @@ import { join } from 'node:path'; import { Worker } from 'node:worker_threads'; import { pluginEvents } from '../../plugin/index.js'; import { t } from '../t.js'; +import { watchPath, unwatchPath } from '../watcher/watcher.js'; try { const plugins = freeCodeCampConfig.tooling?.plugins; diff --git a/.freeCodeCamp/tooling/watcher/watcher.js b/.freeCodeCamp/tooling/watcher/watcher.js index 60d65146..224307d0 100644 --- a/.freeCodeCamp/tooling/watcher/watcher.js +++ b/.freeCodeCamp/tooling/watcher/watcher.js @@ -1,6 +1,49 @@ import path from 'path'; -import { PATHS_TO_IGNORE } from '../hot-reload.js'; -import { ROOT } from '../env.js'; +import { ROOT, freeCodeCampConfig } from '../env.js'; + +const defaultPathsToIgnore = [ + '/.logs/.script_out.log', + '/.logs/.script_in.log', + '/.logs/.temp.log', + '/config/', + 'node_modules/', + '.git/', + 'target/', + 'test-ledger/' +]; + +/** + * Paths following the following convention: + * - If path starts with `/`, it is relative to `ROOT` + * - Else, path is anywhere in tree + * + * **Examples:** + * + * - ["/node_modules/"] - ignore only `node_modules` in ROOT + * - ["node_modules/"] - ignore `node_modules` anywhere in the tree + * + * **NOTE:** Globs are NOT supported + * @type {string[]} + */ +export const PATHS_TO_IGNORE = + freeCodeCampConfig.hotReload?.ignore || defaultPathsToIgnore; + +/** + * Overrides `PATHS_TO_IGNORE`. + * + * Paths following the following convention: + * - If path starts with `/`, it is relative to `ROOT` + * - Else, path is anywhere in tree + * + * **Examples:** + * + * - ["/node_modules/"] - watch only `node_modules` in ROOT + * - ["node_modules/"] - watch `node_modules` anywhere in the tree + * + * **NOTE:** Globs are NOT supported + * @type {string[]} + */ +export const PATHS_TO_WATCH = []; // TODO: // 1. Ensure all paths start with `ROOT` so comparisions work @@ -13,19 +56,31 @@ import { ROOT } from '../env.js'; // - `PATHS_TO_IGNORE` should be relative to `ROOT` // - Must include the full path to the file/directory wanting to be ignored from `ROOT` -export function unwatchPath(pathRelativeToRoot) {} +/** + * + * @param {string} pathRelativeToRoot + */ +export function unwatchPath(pathRelativeToRoot) { + maybeAddRelativePath(PATHS_TO_IGNORE, pathRelativeToRoot); +} -export function watchPath(pathRelativeToRoot) {} +/** + * + * @param {string} pathRelativeToRoot + */ +export function watchPath(pathRelativeToRoot) { + maybeAddRelativePath(PATHS_TO_WATCH, pathRelativeToRoot); +} /** - * Add `pathRelativeToRoot` to `PATHS_TO_IGNORE`, if it is not already **equivalently** added + * Add `pathRelativeToRoot` to `list`, if it is not already **equivalently** added * * **Examples** * * ```js * const PATHS_TO_IGNORE = ["node_modules/"]; * const pathRelativeToRoot = "/node_modules/foo/bar"; - * maybeAddRelativePathToPTI(pathRelativeToRoot); + * maybeAddRelativePath(PATHS_TO_IGNORE,pathRelativeToRoot); * console.log(PATHS_TO_IGNORE); * // ["node_modules/"] * ``` @@ -36,25 +91,26 @@ export function watchPath(pathRelativeToRoot) {} * ```js * const PATHS_TO_IGNORE = ["/node_modules/"]; * const pathRelativeToRoot = "node_modules/foo/bar"; - * maybeAddRelativePathToPTI(pathRelativeToRoot); + * maybeAddRelativePath(PATHS_TO_IGNORE,pathRelativeToRoot); * console.log(PATHS_TO_IGNORE); * // ["/node_modules/", "node_modules/foo/bar"] * ``` * * `node_modules/foo/bar` is added, because `/node_modules/` does not encompass all equivalents. * + * @param {string[]} list * @param {string} pathRelativeToRoot */ -export function maybeAddRelativePathToPTI(pathRelativeToRoot) { - if (!PATHS_TO_IGNORE.some(p => compareEquivalency(p, pathRelativeToRoot))) { +export function maybeAddRelativePath(list, pathRelativeToRoot) { + if (!list.some(p => compareEquivalency(p, pathRelativeToRoot))) { // If reverse equivalency exists for any elements already in PATHS_TO_IGNORE, remove less covered path - PATHS_TO_IGNORE.forEach(p => { + list.forEach(p => { if (compareEquivalency(pathRelativeToRoot, p)) { - PATHS_TO_IGNORE.splice(PATHS_TO_IGNORE.indexOf(p), 1); + list.splice(list.indexOf(p), 1); } }); - PATHS_TO_IGNORE.push(pathRelativeToRoot); + list.push(pathRelativeToRoot); } } @@ -65,7 +121,7 @@ export function maybeAddRelativePathToPTI(pathRelativeToRoot) { * - Allows optional trailing `/` * - If one path is a subset of the other, then they are equivalent * @param {string} covered_path Path already covered - * @param {string} compated_path Path to be covered + * @param {string} compared_path Path to be covered * @returns {boolean} If `covered_path` accounts for `compared_path` */ export function compareEquivalency(covered_path, compared_path) { @@ -89,3 +145,37 @@ export function compareEquivalency(covered_path, compared_path) { return path_compared.includes(path_covered); } + +/** + * + * @param {string[]} list + * @param {string} compared_path + * @returns {boolean} + */ +export function pathIsIn(list, compared_path) { + return list.some(p => compareEquivalency(p, compared_path)); +} + +/** + * + * @param {string} absolutePath + * @returns {boolean} + */ +export function shouldWatch(absolutePath) { + if (pathIsIn(PATHS_TO_WATCH, absolutePath)) { + return true; + } + + if (pathIsIn(PATHS_TO_IGNORE, absolutePath)) { + return false; + } + + return true; +} + +export function resetPathLists() { + const initial_paths_to_ignore = + freeCodeCampConfig.hotReload?.ignore || defaultPathsToIgnore; + PATHS_TO_IGNORE.splice(0, PATHS_TO_IGNORE.length, ...initial_paths_to_ignore); + PATHS_TO_WATCH.splice(0, PATHS_TO_WATCH.length); +} diff --git a/docs/src/testing/test-utilities.md b/docs/src/testing/test-utilities.md index e5f0d16f..9fc63f71 100644 --- a/docs/src/testing/test-utilities.md +++ b/docs/src/testing/test-utilities.md @@ -10,6 +10,7 @@ Many of the exported functions are _convinience wrappers_ around Nodejs' `fs` an - [`getCWD`](#getcwd) - [`getLastCommand`](#getlastcommand) - [`getLastCWD`](#getlastcwd) +- [`getScriptIn`](#getscriptin) - [`getScriptOut`](#getscriptout) - [`getTemp`](#gettemp) - [`getTerminalOutput`](#getterminaloutput) @@ -121,6 +122,26 @@ function getLastCWD(n = 0): Promise; const lastCWD = await __helpers.getLastCWD(); ``` +## `getScriptIn` + +Get the `.logs/.script_in.log` file contents. + +```admonish danger title="Safety" +Throws if file does not exist, or if read permission is denied. +``` + +```typescript +function getScriptIn(): Promise; +``` + +```javascript +const scriptIn = await __helpers.getScriptIn(); +``` + +```admonish note +The output does not include _untyped_ characters. That is, if tab completion is used, the output will not include the final command, but only up to the point of completion. +``` + ## `getScriptOut` Get the `.logs/.script_out.log` file contents. diff --git a/package-lock.json b/package-lock.json index f0d35bc0..21f930ea 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@freecodecamp/freecodecamp-os", - "version": "3.5.1", + "version": "3.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@freecodecamp/freecodecamp-os", - "version": "3.5.1", + "version": "3.6.0", "dependencies": { "chai": "4.4.1", "chokidar": "3.6.0", diff --git a/package.json b/package.json index fdbf5c79..798f3914 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,6 @@ "develop:client": "NODE_ENV=development webpack --mode development --config ./.freeCodeCamp/webpack.config.cjs --watch", "develop:server": "nodemon --watch ./.freeCodeCamp/dist/ --watch ./.freeCodeCamp/tooling/ --watch ./tooling/ --ignore ./config/ ./.freeCodeCamp/tooling/server.js", "start": "npm run build:client && node ./.freeCodeCamp/tooling/server.js", - "test": "node ./.freeCodeCamp/tests/parser.test.js", "prepublishOnly": "npm run build:client" }, "dependencies": { diff --git a/self/curriculum/locales/english/dot-logs.md b/self/curriculum/locales/english/dot-logs.md index a0bc26d4..db55d729 100644 --- a/self/curriculum/locales/english/dot-logs.md +++ b/self/curriculum/locales/english/dot-logs.md @@ -4,6 +4,12 @@ The `.logs/` folder and features ## 0 +```json +{ + "watch": [".logs/.script_in.log"] +} +``` + ### --description-- In a new terminal, start typing `dot logs`, and the tests should run. @@ -13,8 +19,8 @@ In a new terminal, start typing `dot logs`, and the tests should run. Placeholder test. ```js -const scriptOut = await __helpers.getScriptOut(); -assert.match(scriptOut, /dot logs/); +const scriptIn = await __helpers.getScriptInEquivalent(); +assert.match(scriptIn, /dot logs/); ``` ## 1 diff --git a/self/package.json b/self/package.json index db7259a4..8d78d212 100644 --- a/self/package.json +++ b/self/package.json @@ -2,10 +2,13 @@ "name": "self", "private": true, "author": "freeCodeCamp", - "version": "3.4.0", + "version": "3.6.0", "description": "Test repo for @freecodecamp/freecodecamp-os", "scripts": { - "start": "node ./node_modules/@freecodecamp/freecodecamp-os/.freeCodeCamp/tooling/server.js" + "start": "node ./node_modules/@freecodecamp/freecodecamp-os/.freeCodeCamp/tooling/server.js", + "test": "npm run test:parser && npm run test:node", + "test:parser": "node ../.freeCodeCamp/tests/parser.test.js", + "test:node": "node --test ../.freeCodeCamp/tests/watcher.test.js" }, "dependencies": { "@freecodecamp/freecodecamp-os": "../" From 939bd8040ebc6d4ab08ceddd0cbb412745cc4b8f Mon Sep 17 00:00:00 2001 From: Shaun Hamilton Date: Fri, 12 Apr 2024 17:58:43 +0200 Subject: [PATCH 4/4] fix conf for self --- self/curriculum/locales/english/dot-logs.md | 2 +- self/freecodecamp.conf.json | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/self/curriculum/locales/english/dot-logs.md b/self/curriculum/locales/english/dot-logs.md index db55d729..e34099d5 100644 --- a/self/curriculum/locales/english/dot-logs.md +++ b/self/curriculum/locales/english/dot-logs.md @@ -27,7 +27,7 @@ assert.match(scriptIn, /dot logs/); ### --description-- -As +Well done. ### --tests-- diff --git a/self/freecodecamp.conf.json b/self/freecodecamp.conf.json index 36b831da..ee5f751e 100644 --- a/self/freecodecamp.conf.json +++ b/self/freecodecamp.conf.json @@ -40,12 +40,13 @@ }, "hotReload": { "ignore": [ - ".logs/.temp.log", + "/.logs/.script_in.log", + "/.logs/.script_out.log", + "/.logs/.temp.log", "config/", - "/node_modules/", + "node_modules/", ".git/", - "/target/", - "/test-ledger/", + "target/", ".vscode/", "freecodecamp.conf.json" ]