From 7e6613a1a05709df1ef7fe169048ba65638ea275 Mon Sep 17 00:00:00 2001 From: Miguel Serrano Date: Wed, 7 Jun 2023 13:10:02 +0200 Subject: [PATCH] Merge pull request #13327 from overleaf/msm-web-track-changes-cleanup [web] Cleanup track-changes from web GitOrigin-RevId: 8cef709ec5e91e4ffe8cd06826038ed84f36ef67 --- package-lock.json | 286 +--- services/web/.gitignore | 3 - .../Features/Documents/DocumentController.js | 20 +- .../src/Features/History/HistoryController.js | 41 +- .../src/Features/History/HistoryManager.js | 8 - .../src/Features/Project/ProjectController.js | 4 - .../Project/ProjectCreationHandler.js | 13 +- .../Features/Project/ProjectHistoryHandler.js | 67 - services/web/app/src/router.js | 11 - .../web/app/views/project/editor/meta.pug | 1 - services/web/config/settings.defaults.js | 12 +- services/web/frontend/js/ide.js | 7 +- .../app/src/HistoryUpgradeHelper.js | 394 ----- .../app/src/ProjectHistoryController.js | 1324 ----------------- .../web/modules/history-migration/index.js | 1 - .../unit/src/ProjectHistoryControllerTests.js | 346 ----- .../migrate-project-history.snapshot.json | 50 - .../unit/src/data/track-changes-project.zip | Bin 3527 -> 0 bytes services/web/package.json | 1 - services/web/scripts/fix_oversized_docs.js | 1 - .../scripts/history/clean_sl_history_data.js | 60 - .../count_project_history_categories.js | 84 -- services/web/scripts/history/debug_history.js | 328 ---- .../web/scripts/history/downgrade_project.js | 81 - .../web/scripts/history/migrate_history.js | 287 ---- .../history/reset_incorrect_doc_revision.js | 111 -- .../scripts/history/unset_allow_downgrade.js | 133 -- ...rade_none_with_conversion_if_sl_history.js | 208 --- ...one_without_conversion_if_no_sl_history.js | 232 --- .../web/scripts/history/upgrade_project.js | 44 - ...pgrade_v1_with_conversion_if_sl_history.js | 243 --- ...conversion_if_created_after_fph_enabled.js | 165 -- ..._v1_without_conversion_if_no_sl_history.js | 161 -- .../web/scripts/recover_docs_from_redis.js | 1 - .../config/settings.test.defaults.js | 5 - .../src/Documents/DocumentControllerTests.js | 118 -- .../src/History/HistoryControllerTests.js | 73 - .../unit/src/History/HistoryManagerTests.js | 75 +- 38 files changed, 44 insertions(+), 4955 deletions(-) delete mode 100644 services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js delete mode 100644 services/web/modules/history-migration/app/src/ProjectHistoryController.js delete mode 100644 services/web/modules/history-migration/index.js delete mode 100644 services/web/modules/history-migration/test/unit/src/ProjectHistoryControllerTests.js delete mode 100644 services/web/modules/history-migration/test/unit/src/data/migrate-project-history.snapshot.json delete mode 100644 services/web/modules/history-migration/test/unit/src/data/track-changes-project.zip delete mode 100644 services/web/scripts/history/clean_sl_history_data.js delete mode 100644 services/web/scripts/history/count_project_history_categories.js delete mode 100644 services/web/scripts/history/debug_history.js delete mode 100644 services/web/scripts/history/downgrade_project.js delete mode 100644 services/web/scripts/history/migrate_history.js delete mode 100644 services/web/scripts/history/reset_incorrect_doc_revision.js delete mode 100644 services/web/scripts/history/unset_allow_downgrade.js delete mode 100644 services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js delete mode 100644 services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js delete mode 100644 services/web/scripts/history/upgrade_project.js delete mode 100644 services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js delete mode 100644 services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js delete mode 100644 services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js diff --git a/package-lock.json b/package-lock.json index 0280d2f146..6fc95df224 100644 --- a/package-lock.json +++ b/package-lock.json @@ -379,6 +379,7 @@ "version": "3.0.0" }, "libraries/stream-utils": { + "name": "@overleaf/stream-utils", "version": "0.1.0", "license": "AGPL-3.0-only", "devDependencies": { @@ -14620,18 +14621,6 @@ "node": "*" } }, - "node_modules/binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", - "dependencies": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - }, - "engines": { - "node": "*" - } - }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -15036,14 +15025,6 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, - "node_modules/buffer-indexof-polyfill": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", - "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", - "engines": { - "node": ">=0.10" - } - }, "node_modules/buffer-writer": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", @@ -15611,17 +15592,6 @@ "node": ">=4" } }, - "node_modules/chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", - "dependencies": { - "traverse": ">=0.3.0 <0.4" - }, - "engines": { - "node": "*" - } - }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -19247,41 +19217,6 @@ "node": ">=0.10" } }, - "node_modules/duplexer2": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", - "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=", - "dependencies": { - "readable-stream": "^2.0.2" - } - }, - "node_modules/duplexer2/node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "node_modules/duplexer2/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/duplexer2/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", @@ -22682,20 +22617,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/fstream": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", - "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", - "dependencies": { - "graceful-fs": "^4.1.2", - "inherits": "~2.0.0", - "mkdirp": ">=0.5 0", - "rimraf": "2" - }, - "engines": { - "node": ">=0.6" - } - }, "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -27720,11 +27641,6 @@ "node": ">=0.10.0" } }, - "node_modules/listenercount": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", - "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=" - }, "node_modules/listr2": { "version": "3.14.0", "resolved": "https://registry.npmjs.org/listr2/-/listr2-3.14.0.tgz", @@ -35128,11 +35044,6 @@ "node": ">=0.10.0" } }, - "node_modules/setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" - }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -37594,14 +37505,6 @@ "punycode": "^2.1.0" } }, - "node_modules/traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=", - "engines": { - "node": "*" - } - }, "node_modules/ts-dedent": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.2.0.tgz", @@ -38059,55 +37962,6 @@ "node": ">=8" } }, - "node_modules/unzipper": { - "version": "0.10.11", - "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.11.tgz", - "integrity": "sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==", - "dependencies": { - "big-integer": "^1.6.17", - "binary": "~0.3.0", - "bluebird": "~3.4.1", - "buffer-indexof-polyfill": "~1.0.0", - "duplexer2": "~0.1.4", - "fstream": "^1.0.12", - "graceful-fs": "^4.2.2", - "listenercount": "~1.0.1", - "readable-stream": "~2.3.6", - "setimmediate": "~1.0.4" - } - }, - "node_modules/unzipper/node_modules/bluebird": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", - "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=" - }, - "node_modules/unzipper/node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "node_modules/unzipper/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/unzipper/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/update-browserslist-db": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz", @@ -41792,7 +41646,6 @@ "scroll-into-view-if-needed": "^2.2.25", "tsscmp": "^1.0.6", "underscore": "^1.13.1", - "unzipper": "^0.10.11", "utf-8-validate": "^5.0.2", "uuid": "^3.0.1", "valid-data-url": "^2.0.0", @@ -50588,7 +50441,6 @@ "tsscmp": "^1.0.6", "typescript": "^4.5.5", "underscore": "^1.13.1", - "unzipper": "^0.10.11", "utf-8-validate": "^5.0.2", "uuid": "^3.0.1", "val-loader": "^5.0.1", @@ -56100,15 +55952,6 @@ "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.2.tgz", "integrity": "sha512-GAcQvbpsM0pUb0zw1EI0KhQEZ+lRwR5fYaAp3vPOYuP7aDvGy6cVN6XHLauvF8SOga2y0dcLcjt3iQDTSEliyw==" }, - "binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", - "requires": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - } - }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -56444,11 +56287,6 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, - "buffer-indexof-polyfill": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", - "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==" - }, "buffer-writer": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", @@ -56881,14 +56719,6 @@ "superagent": "^3.7.0" } }, - "chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", - "requires": { - "traverse": ">=0.3.0 <0.4" - } - }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -59543,43 +59373,6 @@ "nan": "^2.14.0" } }, - "duplexer2": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", - "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=", - "requires": { - "readable-stream": "^2.0.2" - }, - "dependencies": { - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - } - } - }, "duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", @@ -62206,17 +61999,6 @@ "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", "optional": true }, - "fstream": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", - "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", - "requires": { - "graceful-fs": "^4.1.2", - "inherits": "~2.0.0", - "mkdirp": ">=0.5 0", - "rimraf": "2" - } - }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -66088,11 +65870,6 @@ "repeat-string": "^1.5.2" } }, - "listenercount": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", - "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=" - }, "listr2": { "version": "3.14.0", "resolved": "https://registry.npmjs.org/listr2/-/listr2-3.14.0.tgz", @@ -72108,11 +71885,6 @@ "to-object-path": "^0.3.0" } }, - "setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" - }, "setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -74077,11 +73849,6 @@ "punycode": "^2.1.0" } }, - "traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=" - }, "ts-dedent": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.2.0.tgz", @@ -74423,57 +74190,6 @@ "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", "integrity": "sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==" }, - "unzipper": { - "version": "0.10.11", - "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.11.tgz", - "integrity": "sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==", - "requires": { - "big-integer": "^1.6.17", - "binary": "~0.3.0", - "bluebird": "~3.4.1", - "buffer-indexof-polyfill": "~1.0.0", - "duplexer2": "~0.1.4", - "fstream": "^1.0.12", - "graceful-fs": "^4.2.2", - "listenercount": "~1.0.1", - "readable-stream": "~2.3.6", - "setimmediate": "~1.0.4" - }, - "dependencies": { - "bluebird": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", - "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - } - } - }, "update-browserslist-db": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz", diff --git a/services/web/.gitignore b/services/web/.gitignore index ad9693e9a8..f49076694e 100644 --- a/services/web/.gitignore +++ b/services/web/.gitignore @@ -88,9 +88,6 @@ cypress/videos/ cypress/downloads/ cypress/results/ -# Test fixture zip -!modules/history-migration/test/unit/src/data/track-changes-project.zip - # Ace themes for conversion frontend/js/features/source-editor/themes/ace/ diff --git a/services/web/app/src/Features/Documents/DocumentController.js b/services/web/app/src/Features/Documents/DocumentController.js index 0e4958c19c..57daba7497 100644 --- a/services/web/app/src/Features/Documents/DocumentController.js +++ b/services/web/app/src/Features/Documents/DocumentController.js @@ -51,21 +51,11 @@ function getDocument(req, res, next) { plainTextResponse(res, lines.join('\n')) } else { const projectHistoryId = _.get(project, 'overleaf.history.id') - const projectHistoryDisplay = _.get( - project, - 'overleaf.history.display' - ) - const sendToBothHistorySystems = _.get( - project, - 'overleaf.history.allowDowngrade' - ) - // if project has been switched but has 'allowDowngrade' set - // then leave projectHistoryType undefined to (temporarily) - // continue sending updates to both SL and full project history - const projectHistoryType = - projectHistoryDisplay && !sendToBothHistorySystems - ? 'project-history' - : undefined // for backwards compatibility, don't send anything if the project is still on track-changes + + // all projects are now migrated to Full Project History, keeping the field + // for API compatibility + const projectHistoryType = 'project-history' + res.json({ lines, version, diff --git a/services/web/app/src/Features/History/HistoryController.js b/services/web/app/src/Features/History/HistoryController.js index 6fec20ac63..7a64fc8ef5 100644 --- a/services/web/app/src/Features/History/HistoryController.js +++ b/services/web/app/src/Features/History/HistoryController.js @@ -16,35 +16,9 @@ const { prepareZipAttachment } = require('../../infrastructure/Response') const Features = require('../../infrastructure/Features') module.exports = HistoryController = { - selectHistoryApi(req, res, next) { - const { Project_id: projectId } = req.params - // find out which type of history service this project uses - ProjectDetailsHandler.getDetails(projectId, function (err, project) { - if (err) { - return next(err) - } - const history = project.overleaf && project.overleaf.history - if (history && history.id && history.display) { - req.useProjectHistory = true - } else { - req.useProjectHistory = false - } - next() - }) - }, - - ensureProjectHistoryEnabled(req, res, next) { - if (req.useProjectHistory) { - next() - } else { - res.sendStatus(404) - } - }, - proxyToHistoryApi(req, res, next) { const userId = SessionManager.getLoggedInUserId(req.session) - const url = - HistoryController.buildHistoryServiceUrl(req.useProjectHistory) + req.url + const url = settings.apis.project_history.url + req.url const getReq = request({ url, @@ -65,8 +39,7 @@ module.exports = HistoryController = { proxyToHistoryApiAndInjectUserDetails(req, res, next) { const userId = SessionManager.getLoggedInUserId(req.session) - const url = - HistoryController.buildHistoryServiceUrl(req.useProjectHistory) + req.url + const url = settings.apis.project_history.url + req.url HistoryController._makeRequest( { url, @@ -90,16 +63,6 @@ module.exports = HistoryController = { ) }, - buildHistoryServiceUrl(useProjectHistory) { - // choose a history service, either document-level (trackchanges) - // or project-level (project_history) - if (useProjectHistory) { - return settings.apis.project_history.url - } else { - return settings.apis.trackchanges.url - } - }, - resyncProjectHistory(req, res, next) { // increase timeout to 6 minutes res.setTimeout(6 * 60 * 1000) diff --git a/services/web/app/src/Features/History/HistoryManager.js b/services/web/app/src/Features/History/HistoryManager.js index 9748e82506..451a74444c 100644 --- a/services/web/app/src/Features/History/HistoryManager.js +++ b/services/web/app/src/Features/History/HistoryManager.js @@ -5,14 +5,6 @@ const OError = require('@overleaf/o-error') const UserGetter = require('../User/UserGetter') async function initializeProject(projectId) { - if ( - !( - settings.apis.project_history && - settings.apis.project_history.initializeHistoryForNewProjects - ) - ) { - return null - } const response = await fetch(`${settings.apis.project_history.url}/project`, { method: 'POST', headers: { diff --git a/services/web/app/src/Features/Project/ProjectController.js b/services/web/app/src/Features/Project/ProjectController.js index 7b4608011f..85bb60584f 100644 --- a/services/web/app/src/Features/Project/ProjectController.js +++ b/services/web/app/src/Features/Project/ProjectController.js @@ -1064,10 +1064,6 @@ const ProjectController = { editorThemes: THEME_LIST, legacyEditorThemes: LEGACY_THEME_LIST, maxDocLength: Settings.max_doc_length, - useV2History: - project.overleaf && - project.overleaf.history && - Boolean(project.overleaf.history.display), brandVariation, allowedImageNames, gitBridgePublicBaseUrl: Settings.gitBridgePublicBaseUrl, diff --git a/services/web/app/src/Features/Project/ProjectCreationHandler.js b/services/web/app/src/Features/Project/ProjectCreationHandler.js index 687993a7fa..cbb150447d 100644 --- a/services/web/app/src/Features/Project/ProjectCreationHandler.js +++ b/services/web/app/src/Features/Project/ProjectCreationHandler.js @@ -169,15 +169,10 @@ async function _createBlankProject( } } - // only display full project history when the project has the overleaf history id attribute - // (to allow scripted creation of projects without full project history) - const historyId = project.overleaf.history.id - if ( - Settings.apis.project_history.displayHistoryForNewProjects && - historyId != null - ) { - project.overleaf.history.display = true - } + // All the projects are initialised with Full Project History. This property + // is still set for backwards compatibility: Server Pro requires all projects + // have it set to `true` since SP 4.0 + project.overleaf.history.display = true if (Settings.currentImageName) { // avoid clobbering any imageName already set in attributes (e.g. importedImageName) diff --git a/services/web/app/src/Features/Project/ProjectHistoryHandler.js b/services/web/app/src/Features/Project/ProjectHistoryHandler.js index 939d87ba07..572b06f378 100644 --- a/services/web/app/src/Features/Project/ProjectHistoryHandler.js +++ b/services/web/app/src/Features/Project/ProjectHistoryHandler.js @@ -35,73 +35,6 @@ const ProjectHistoryHandler = { }) }, - unsetHistory(projectId, callback) { - Project.updateOne( - { _id: projectId }, - { $unset: { 'overleaf.history': true } }, - callback - ) - }, - - upgradeHistory(projectId, allowDowngrade, callback) { - // project must have an overleaf.history.id before allowing display of new history - Project.updateOne( - { _id: projectId, 'overleaf.history.id': { $exists: true } }, - { - 'overleaf.history.display': true, - 'overleaf.history.upgradedAt': new Date(), - 'overleaf.history.allowDowngrade': allowDowngrade, - }, - function (err, result) { - if (err) { - return callback(err) - } - // return an error if overleaf.history.id wasn't present - if (result.matchedCount === 0) { - return callback(new Error('history not upgraded')) - } - callback() - } - ) - }, - - downgradeHistory(projectId, callback) { - Project.updateOne( - { _id: projectId, 'overleaf.history.upgradedAt': { $exists: true } }, - { - 'overleaf.history.display': false, - $unset: { 'overleaf.history.upgradedAt': 1 }, - }, - function (err, result) { - if (err) { - return callback(err) - } - if (result.matchedCount === 0) { - return callback(new Error('history not downgraded')) - } - callback() - } - ) - }, - - setMigrationArchiveFlag(projectId, callback) { - Project.updateOne( - { _id: projectId, version: { $exists: true } }, - { - 'overleaf.history.zipFileArchivedInProject': true, - }, - function (err, result) { - if (err) { - return callback(err) - } - if (result.matchedCount === 0) { - return callback(new Error('migration flag not set')) - } - callback() - } - ) - }, - ensureHistoryExistsForProject(projectId, callback) { // We can only set a history id for a project that doesn't have one. The // history id is cached in the project history service, and changing an diff --git a/services/web/app/src/router.js b/services/web/app/src/router.js index ecac532c86..8c0c1fb74f 100644 --- a/services/web/app/src/router.js +++ b/services/web/app/src/router.js @@ -711,34 +711,29 @@ function initialize(webRouter, privateApiRouter, publicApiRouter) { '/project/:Project_id/updates', AuthorizationMiddleware.blockRestrictedUserFromProject, AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.selectHistoryApi, HistoryController.proxyToHistoryApiAndInjectUserDetails ) webRouter.get( '/project/:Project_id/doc/:doc_id/diff', AuthorizationMiddleware.blockRestrictedUserFromProject, AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.selectHistoryApi, HistoryController.proxyToHistoryApi ) webRouter.get( '/project/:Project_id/diff', AuthorizationMiddleware.blockRestrictedUserFromProject, AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.selectHistoryApi, HistoryController.proxyToHistoryApiAndInjectUserDetails ) webRouter.get( '/project/:Project_id/filetree/diff', AuthorizationMiddleware.blockRestrictedUserFromProject, AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.selectHistoryApi, HistoryController.proxyToHistoryApi ) webRouter.post( '/project/:Project_id/doc/:doc_id/version/:version_id/restore', AuthorizationMiddleware.ensureUserCanWriteProjectContent, - HistoryController.selectHistoryApi, HistoryController.proxyToHistoryApi ) webRouter.post( @@ -768,22 +763,16 @@ function initialize(webRouter, privateApiRouter, publicApiRouter) { '/project/:Project_id/labels', AuthorizationMiddleware.blockRestrictedUserFromProject, AuthorizationMiddleware.ensureUserCanReadProject, - HistoryController.selectHistoryApi, - HistoryController.ensureProjectHistoryEnabled, HistoryController.getLabels ) webRouter.post( '/project/:Project_id/labels', AuthorizationMiddleware.ensureUserCanWriteProjectContent, - HistoryController.selectHistoryApi, - HistoryController.ensureProjectHistoryEnabled, HistoryController.createLabel ) webRouter.delete( '/project/:Project_id/labels/:label_id', AuthorizationMiddleware.ensureUserCanWriteProjectContent, - HistoryController.selectHistoryApi, - HistoryController.ensureProjectHistoryEnabled, HistoryController.deleteLabel ) diff --git a/services/web/app/views/project/editor/meta.pug b/services/web/app/views/project/editor/meta.pug index c66b6b1bdd..bd6410ae18 100644 --- a/services/web/app/views/project/editor/meta.pug +++ b/services/web/app/views/project/editor/meta.pug @@ -1,4 +1,3 @@ -meta(name="ol-useV2History" data-type="boolean" content=useV2History) meta(name="ol-project_id" content=project_id) meta(name="ol-projectName" content=projectName) meta(name="ol-userSettings" data-type="json" content=userSettings) diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 272cbcc35e..c7174f6239 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -186,9 +186,6 @@ module.exports = { url: `http://${process.env.SPELLING_HOST || 'localhost'}:3005`, host: process.env.SPELLING_HOST, }, - trackchanges: { - url: `http://${process.env.TRACK_CHANGES_HOST || 'localhost'}:3015`, - }, docstore: { url: `http://${process.env.DOCSTORE_HOST || 'localhost'}:3016`, pubUrl: `http://${process.env.DOCSTORE_HOST || 'localhost'}:3016`, @@ -207,8 +204,6 @@ module.exports = { }, project_history: { sendProjectStructureOps: true, - initializeHistoryForNewProjects: true, - displayHistoryForNewProjects: true, url: `http://${process.env.PROJECT_HISTORY_HOST || 'localhost'}:3054`, }, realTime: { @@ -810,12 +805,7 @@ module.exports = { oauth2Server: [], }, - moduleImportSequence: [ - 'launchpad', - 'server-ce-scripts', - 'user-activate', - 'history-migration', - ], + moduleImportSequence: ['launchpad', 'server-ce-scripts', 'user-activate'], csp: { enabled: process.env.CSP_ENABLED === 'true', diff --git a/services/web/frontend/js/ide.js b/services/web/frontend/js/ide.js index 8ef2a1dbde..e7ae4f51dd 100644 --- a/services/web/frontend/js/ide.js +++ b/services/web/frontend/js/ide.js @@ -22,7 +22,6 @@ import LoadingManager from './ide/LoadingManager' import ConnectionManager from './ide/connection/ConnectionManager' import EditorManager from './ide/editor/EditorManager' import OnlineUsersManager from './ide/online-users/OnlineUsersManager' -import HistoryManager from './ide/history/HistoryManager' import HistoryV2Manager from './ide/history/HistoryV2Manager' import PermissionsManager from './ide/permissions/PermissionsManager' import BinaryFilesManager from './ide/binary-files/BinaryFilesManager' @@ -213,11 +212,7 @@ App.controller( eventTracking ) ide.onlineUsersManager = new OnlineUsersManager(ide, $scope) - if (window.data.useV2History) { - ide.historyManager = new HistoryV2Manager(ide, $scope, localStorage) - } else { - ide.historyManager = new HistoryManager(ide, $scope) - } + ide.historyManager = new HistoryV2Manager(ide, $scope, localStorage) ide.permissionsManager = new PermissionsManager(ide, $scope) ide.binaryFilesManager = new BinaryFilesManager(ide, $scope) ide.metadataManager = new MetadataManager(ide, $scope, metadata) diff --git a/services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js b/services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js deleted file mode 100644 index 07af2cab38..0000000000 --- a/services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js +++ /dev/null @@ -1,394 +0,0 @@ -const { ObjectId } = require('mongodb') -const { - db, - READ_PREFERENCE_SECONDARY, -} = require('../../../../app/src/infrastructure/mongodb') -const Settings = require('@overleaf/settings') - -const ProjectHistoryHandler = require('../../../../app/src/Features/Project/ProjectHistoryHandler') -const HistoryManager = require('../../../../app/src/Features/History/HistoryManager') -const ProjectHistoryController = require('./ProjectHistoryController') -const ProjectEntityHandler = require('../../../../app/src/Features/Project/ProjectEntityHandler') -const ProjectEntityUpdateHandler = require('../../../../app/src/Features/Project/ProjectEntityUpdateHandler') -const DocumentUpdaterHandler = require('../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler') - -// Timestamp of when 'Enable history for SL in background' release -const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = - Settings.apis.project_history?.idWhenFullProjectHistoryEnabled // was '5a8d8a370000000000000000' -const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED = - ID_WHEN_FULL_PROJECT_HISTORY_ENABLED - ? new ObjectId(ID_WHEN_FULL_PROJECT_HISTORY_ENABLED).getTimestamp() - : null - -async function countProjects(query = {}) { - const count = await db.projects.countDocuments(query) - return count -} - -async function countDocHistory(query = {}) { - const count = await db.docHistory.countDocuments(query) - return count -} - -async function findProjects(query = {}, projection = {}) { - const projects = await db.projects.find(query).project(projection).toArray() - return projects -} - -async function determineProjectHistoryType(project) { - if (project.overleaf && project.overleaf.history) { - if (project.overleaf.history.upgradeFailed) { - return 'UpgradeFailed' - } - if (project.overleaf.history.conversionFailed) { - return 'ConversionFailed' - } - } - if ( - project.overleaf && - project.overleaf.history && - project.overleaf.history.id - ) { - if (project.overleaf.history.display) { - // v2: full project history, do nothing - return 'V2' - } else { - if (projectCreatedAfterFullProjectHistoryEnabled(project)) { - // IF project initialised after full project history enabled for all projects - // THEN project history should contain all information we need, without intervention - return 'V1WithoutConversion' - } else { - // ELSE SL history may predate full project history - // THEN delete full project history and convert their SL history to full project history - // -- - // TODO: how to verify this, can get rough start date of SL history, but not full project history - const preserveHistory = await shouldPreserveHistory(project) - const anyDocHistory = await anyDocHistoryExists(project) - const anyDocHistoryIndex = await anyDocHistoryIndexExists(project) - if (preserveHistory) { - if (anyDocHistory || anyDocHistoryIndex) { - // if SL history exists that we need to preserve, then we must convert - return 'V1WithConversion' - } else { - // otherwise just upgrade without conversion - return 'V1WithoutConversion' - } - } else { - // if preserveHistory false, then max 7 days of SL history - // but v1 already record to both histories, so safe to upgrade - return 'V1WithoutConversion' - } - } - } - } else { - const preserveHistory = await shouldPreserveHistory(project) - const anyDocHistory = await anyDocHistoryExists(project) - const anyDocHistoryIndex = await anyDocHistoryIndexExists(project) - if (anyDocHistory || anyDocHistoryIndex) { - // IF there is SL history -> - if (preserveHistory) { - // that needs to be preserved: - // THEN initialise full project history and convert SL history to full project history - return 'NoneWithConversion' - } else { - return 'NoneWithTemporaryHistory' - } - } else { - // ELSE there is not any SL history -> - // THEN initialise full project history and sync with current content - return 'NoneWithoutConversion' - } - } -} - -async function upgradeProject(project, options) { - const historyType = await determineProjectHistoryType(project) - if (historyType === 'V2') { - return { historyType, upgraded: true } - } - const upgradeFn = getUpgradeFunctionForType(historyType) - if (!upgradeFn) { - return { error: 'unsupported history type' } - } - if (options.forceClean) { - try { - const projectId = project._id - // delete any existing history stored in the mongo backend - await HistoryManager.promises.deleteProject(projectId, projectId) - // unset overleaf.history.id to prevent the migration script from failing on checks - await db.projects.updateOne( - { _id: projectId }, - { $unset: { 'overleaf.history.id': '' } } - ) - } catch (err) { - // failed to delete existing history, but we can try to continue - } - } - const result = await upgradeFn(project, options) - result.historyType = historyType - return result -} - -// Do upgrades/conversion: - -function getUpgradeFunctionForType(historyType) { - return UpgradeFunctionMapping[historyType] -} - -const UpgradeFunctionMapping = { - NoneWithoutConversion: doUpgradeForNoneWithoutConversion, - UpgradeFailed: doUpgradeForNoneWithoutConversion, - ConversionFailed: doUpgradeForNoneWithConversion, - V1WithoutConversion: doUpgradeForV1WithoutConversion, - V1WithConversion: doUpgradeForV1WithConversion, - NoneWithConversion: doUpgradeForNoneWithConversion, - NoneWithTemporaryHistory: doUpgradeForNoneWithConversion, -} - -async function doUpgradeForV1WithoutConversion(project) { - await db.projects.updateOne( - { _id: project._id }, - { - $set: { - 'overleaf.history.display': true, - 'overleaf.history.upgradedAt': new Date(), - 'overleaf.history.upgradeReason': `v1-without-sl-history`, - }, - } - ) - return { upgraded: true } -} - -async function doUpgradeForV1WithConversion(project) { - const result = {} - const projectId = project._id - // migrateProjectHistory expects project id as a string - const projectIdString = project._id.toString() - try { - // We treat these essentially as None projects, the V1 history is irrelevant, - // so we will delete it, and do a conversion as if we're a None project - await ProjectHistoryController.deleteProjectHistory(projectIdString) - await ProjectHistoryController.migrateProjectHistory(projectIdString) - } catch (err) { - // if migrateProjectHistory fails, it cleans up by deleting - // the history and unsetting the history id - // therefore a failed project will still look like a 'None with conversion' project - result.error = err - await db.projects.updateOne( - { _id: projectId }, - { - $set: { - 'overleaf.history.conversionFailed': true, - }, - } - ) - return result - } - await db.projects.updateOne( - { _id: projectId }, - { - $set: { - 'overleaf.history.upgradeReason': `v1-with-conversion`, - }, - $unset: { - 'overleaf.history.upgradeFailed': true, - 'overleaf.history.conversionFailed': true, - }, - } - ) - result.upgraded = true - return result -} - -async function doUpgradeForNoneWithoutConversion(project) { - const result = {} - const projectId = project._id - try { - // Logic originally from ProjectHistoryHandler.ensureHistoryExistsForProject - // However sends a force resync project to project history instead - // of a resync request to doc-updater - let historyId = await ProjectHistoryHandler.promises.getHistoryId(projectId) - if (historyId == null) { - historyId = await HistoryManager.promises.initializeProject(projectId) - if (historyId != null) { - await ProjectHistoryHandler.promises.setHistoryId(projectId, historyId) - } - } - // tell document updater to clear the docs, they will be reloaded with any new history id - await DocumentUpdaterHandler.promises.flushProjectToMongoAndDelete( - projectId - ) - // now resync the project - await HistoryManager.promises.resyncProject(projectId, { - force: true, - origin: { kind: 'history-migration' }, - }) - await HistoryManager.promises.flushProject(projectId) - } catch (err) { - result.error = err - await db.projects.updateOne( - { _id: project._id }, - { - $set: { - 'overleaf.history.upgradeFailed': true, - }, - } - ) - return result - } - await db.projects.updateOne( - { _id: project._id }, - { - $set: { - 'overleaf.history.display': true, - 'overleaf.history.upgradedAt': new Date(), - 'overleaf.history.upgradeReason': `none-without-conversion`, - }, - } - ) - result.upgraded = true - return result -} - -async function doUpgradeForNoneWithConversion(project, options = {}) { - const result = {} - const projectId = project._id - // migrateProjectHistory expects project id as a string - const projectIdString = project._id.toString() - try { - if (options.convertLargeDocsToFile) { - result.convertedDocCount = await convertLargeDocsToFile( - projectId, - options.userId - ) - } - await ProjectHistoryController.migrateProjectHistory( - projectIdString, - options.migrationOptions - ) - } catch (err) { - // if migrateProjectHistory fails, it cleans up by deleting - // the history and unsetting the history id - // therefore a failed project will still look like a 'None with conversion' project - result.error = err - // We set a failed flag so future runs of the script don't automatically retry - await db.projects.updateOne( - { _id: projectId }, - { - $set: { - 'overleaf.history.conversionFailed': true, - }, - } - ) - return result - } - await db.projects.updateOne( - { _id: projectId }, - { - $set: { - 'overleaf.history.upgradeReason': - `none-with-conversion` + options.reason ? `/${options.reason}` : ``, - }, - $unset: { - 'overleaf.history.upgradeFailed': true, - 'overleaf.history.conversionFailed': true, - }, - } - ) - result.upgraded = true - return result -} - -// Util - -function projectCreatedAfterFullProjectHistoryEnabled(project) { - if (DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED == null) { - return false - } else { - return ( - project._id.getTimestamp() >= DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED - ) - } -} - -async function shouldPreserveHistory(project) { - return await db.projectHistoryMetaData.findOne( - { - $and: [ - { project_id: { $eq: project._id } }, - { preserveHistory: { $eq: true } }, - ], - }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) -} - -async function anyDocHistoryExists(project) { - return await db.docHistory.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: READ_PREFERENCE_SECONDARY, - } - ) -} - -async function anyDocHistoryIndexExists(project) { - return await db.docHistoryIndex.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: READ_PREFERENCE_SECONDARY, - } - ) -} - -async function convertLargeDocsToFile(projectId, userId) { - const docs = await ProjectEntityHandler.promises.getAllDocs(projectId) - let convertedDocCount = 0 - for (const doc of Object.values(docs)) { - const sizeBound = JSON.stringify(doc.lines) - if (docIsTooLarge(sizeBound, doc.lines, Settings.max_doc_length)) { - await ProjectEntityUpdateHandler.promises.convertDocToFile( - projectId, - doc._id, - userId, - null - ) - convertedDocCount++ - } - } - return convertedDocCount -} - -// check whether the total size of the document in characters exceeds the -// maxDocLength. -// -// Copied from document-updater: -// https://github.com/overleaf/internal/blob/74adfbebda5f3c2c37d9937f0db5c4106ecde492/services/document-updater/app/js/Limits.js#L18 -function docIsTooLarge(estimatedSize, lines, maxDocLength) { - if (estimatedSize <= maxDocLength) { - return false // definitely under the limit, no need to calculate the total size - } - // calculate the total size, bailing out early if the size limit is reached - let size = 0 - for (const line of lines) { - size += line.length + 1 // include the newline - if (size > maxDocLength) return true - } - // since we didn't hit the limit in the loop, the document is within the allowed length - return false -} - -module.exports = { - countProjects, - countDocHistory, - findProjects, - determineProjectHistoryType, - getUpgradeFunctionForType, - upgradeProject, - convertLargeDocsToFile, - anyDocHistoryExists, - anyDocHistoryIndexExists, - doUpgradeForNoneWithConversion, -} diff --git a/services/web/modules/history-migration/app/src/ProjectHistoryController.js b/services/web/modules/history-migration/app/src/ProjectHistoryController.js deleted file mode 100644 index 1f0a78e032..0000000000 --- a/services/web/modules/history-migration/app/src/ProjectHistoryController.js +++ /dev/null @@ -1,1324 +0,0 @@ -const _ = require('lodash') -const settings = require('@overleaf/settings') -const OError = require('@overleaf/o-error') -const fs = require('fs') -const fse = require('fs-extra') -const { ObjectId } = require('mongodb') -const request = require('request') -const { pipeline } = require('stream') -const unzipper = require('unzipper') -const util = require('util') -const logger = require('@overleaf/logger') -const path = require('path') -const { - FileTooLargeError, - InvalidNameError, -} = require('../../../../app/src/Features/Errors/Errors') -const FilestoreHandler = require('../../../../app/src/Features/FileStore/FileStoreHandler') -const ProjectGetter = require('../../../../app/src/Features/Project/ProjectGetter') -const RedisWrapper = require('../../../../app/src/infrastructure/RedisWrapper') -const HistoryManager = require('../../../../app/src/Features/History/HistoryManager') -const ProjectHistoryHandler = require('../../../../app/src/Features/Project/ProjectHistoryHandler') -const ProjectUpdateHandler = require('../../../../app/src/Features/Project/ProjectUpdateHandler') -const DocumentUpdaterHandler = require('../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler') -const ProjectEntityHandler = require('../../../../app/src/Features/Project/ProjectEntityHandler') -const ProjectEntityUpdateHandler = require('../../../../app/src/Features/Project/ProjectEntityUpdateHandler') -const SafePath = require('../../../../app/src/Features/Project/SafePath') -const { DeletedFile } = require('../../../../app/src/models/DeletedFile') -const { Doc } = require('../../../../app/src/models/Doc') -const { - iterablePaths, -} = require('../../../../app/src/Features/Project/IterablePath') - -const rclient = RedisWrapper.client('project_history_migration') - -module.exports = { deleteProjectHistory, migrateProjectHistory } - -/** - * @typedef {Object} UpdateMeta - * @property {string | null} user_id the id of the user that performed the update - * @property {number} ts the timestamp of the update - */ - -/** - * @typedef {UpdateMeta} EditDocUpdateMeta - * @property {string | null} user_id - * @property {number} ts - * @property {string} pathname the doc pathname - * @property {number} doc_length the length of the doc - */ - -/** - * @typedef {Object} Update - * @property {string} pathname the path in the file tree - * @property {UpdateMeta} meta - // * @property {string} version a two-part version. The first part is the project version after the updates, as recorded in Mongo. The second part is a counter that increments for each update in this batch. - * @property {string} projectHistoryId the v1 history id for this project - * @property {number} v - */ - -/** - * @typedef {Update} FileUpdate - * @property {string} pathname - * @property {UpdateMeta} meta - * @property {string} projectHistoryId - * @property {number} v - * @property {string} file - */ - -/** - * @typedef {FileUpdate} AddFileUpdate - * @property {string} pathname - * @property {UpdateMeta} meta - * @property {string} projectHistoryId - * @property {number} v - * @property {string} file - * @property {string} url - */ - -/** - * @typedef {Update} DocUpdate - * @property {UpdateMeta} meta - * @property {string} projectHistoryId - * @property {number} v - * @property {string} doc - */ - -/** - * @typedef {DocUpdate} AddDocUpdate - * @property {string} pathname - * @property {UpdateMeta} meta - * @property {string} projectHistoryId - * @property {number} v - * @property {string} doc - * @property {string} docLines - * @property {string} docLinesId - * @property {boolean} contentStored - */ - -/** - * @typedef {DocUpdate} EditDocUpdate - * @property {EditDocUpdateMeta} meta - * @property {string} projectHistoryId - * @property {number} v - * @property {number} lastV - * @property {string} doc - * @property {Array} op - */ - -/** - * @typedef {AddDocUpdate | AddFileUpdate} AddUpdate - */ - -/** - * @typedef {DocUpdate | FileUpdate} DeleteUpdate - * @property {string} pathname - * @property {UpdateMeta} meta - * @property {string} projectHistoryId - * @property {number} v - * @property {string} doc - * @property {string} new_pathname - */ - -/** - * @typedef {Update} EditDocUpdateStub - * @property {true} stub - * @property {string} path - * @property {string} pathname - * @property {number} v - * @property {number} doc_length - */ - -/** - * @typedef {AddUpdate | DeleteUpdate | EditDocUpdate | EditDocUpdateStub } AnyUpdate - */ - -/** - * @typedef {Object} Project - * @property {string} _id the id of the user that performed the update - * @property {Object} overleaf - */ - -/** - * @typedef ManifestUpdate - * @property {string} path - * @property {number} doc_length - * @property {number} ts - * @property {number} version - */ - -/** - * @typedef ManifestContent - * @property {number} start - */ - -/** - * @typedef ManifestDoc - * @property {string} id - * @property {ManifestContent} content - * @property {Array} updates - */ - -/** - * @typedef {Object} Manifest - * @property {string} projectId - * @property {Array} docs - */ - -/** - * @typedef Entity - * @property {string} type - * @property {string} path - * @property {string} docLines - * @property {string} deletedAt - * @property {boolean} deleted - */ - -/** - * Iterate recursively through the folders in project.rootFolder, - * building a map of all the docs (with content as a docLines string) - * and files (with content as a filestore URL). - * - * @param {Object} project - * @returns {Promise>} - */ -async function processRootFolder(project) { - const entities = new Map() - - async function processFolder(folder, root = '') { - for (const item of iterablePaths(folder, 'docs')) { - const doc = await Doc.findOne( - item._id, - // only read the fields we need to save memory - { _id: 1, inS3: 1, lines: 1, name: 1 } - ).lean() - - // skip malformed doc entries - if (!doc?._id) { - logger.warn({ doc }, 'skipping doc with missing id') - continue - } - const id = doc._id.toString() - const docIsInS3 = !!doc.inS3 - let docLines - - if (docIsInS3) { - const docPeek = await ProjectEntityHandler.promises.getDoc( - project._id, - item._id, - { peek: true } - ) - docLines = docPeek.lines - } else { - docLines = doc.lines - } - - if (!docLines) { - throw new Error(`no doc lines for doc ${id} (inS3: ${docIsInS3})`) - } - - entities.set(id, { - path: `${root}/${item.name}`, // NOTE: not doc.name, which is "new doc", - type: 'doc', - docLines: docLines.join('\n'), - }) - } - - for (const item of iterablePaths(folder, 'fileRefs')) { - const path = `${root}/${item.name}` - - // skip malformed file entries - if (!item?._id) { - logger.warn({ item }, 'skipping fileRef with missing id') - continue - } - const id = item._id.toString() - - entities.set(id, { - path, - type: 'file', - url: FilestoreHandler._buildUrl(project._id.toString(), id), - }) - } - - for (const subfolder of iterablePaths(folder, 'folders')) { - const path = `${root}/${subfolder.name}` - await processFolder(subfolder, path) - } - } - - for (const folder of project.rootFolder) { - await processFolder(folder) - } - - return entities -} - -/** - * Read docs deleted from a project, from the Doc collection, - * and add them to the entities map with the content in a docLines string. - * - * These entities have a `deleted` property set to `true` and a `deletedAt` date. - * - * @param {Map} entities - * @param {string} projectId - * @returns {Promise} - */ -async function readDeletedDocs(entities, projectId) { - // NOTE: could call DocstoreManager.promises.getAllDeletedDocs(projectId) instead - - // Look for all docs, since some deleted docs are found in track-changes manifest, - // but do not have deleted flag set for reasons that are unclear - // (we will not add docs to entities if they were previously added by processRootFolder) - const deletedDocsCursor = Doc.find( - { - project_id: ObjectId(projectId), - }, - // only read the fields we need to save memory - { _id: 1, inS3: 1, lines: 1, name: 1, deletedAt: 1 } - ) - .lean() - .cursor() - for await (const doc of deletedDocsCursor) { - // skip malformed deleted doc entries - if (!doc?._id) { - logger.warn({ doc }, 'skipping deleted doc with missing id') - continue - } - const id = doc._id.toString() - // Skip doc if we already have an entry in entities - if (!entities.has(id)) { - const docIsInS3 = !!doc.inS3 - let docLines - - if (docIsInS3) { - const docPeek = await ProjectEntityHandler.promises.getDoc( - ObjectId(projectId), - doc._id, - { peek: true } - ) - docLines = docPeek.lines - } else { - docLines = doc.lines - } - - if (!docLines) { - throw new Error(`no doc lines for doc ${id} (inS3: ${docIsInS3})`) - } - - // const ts = Number( - // doc.deletedAt ? new Date(doc.deletedAt) : Date.now() - // ) - - if (doc.name && !SafePath.isCleanFilename(doc.name)) { - const newName = SafePath.clean(doc.name) - logger.warn( - { projectId, docId: id, origName: doc.name, newName }, - 'renaming invalid deleted file' - ) - doc.name = newName - } - - entities.set(id, { - // NOTE: adding the doc id to the file path to avoid collisions - path: `/_deleted/${id}/${doc.name}`, - name: doc.name || 'unnamed', // fallback for improperly deleted docs - deleted: true, - type: 'doc', - deletedAt: doc.deletedAt, - docLines: docLines.join('\n'), - }) - } - } -} - -/** - * Read files deleted from a project, from the DeletedFile collection, - * and add them to the entities map. - * - * These entities have a `deleted` property set to `true` and a `deletedAt` date. - * The url is built later, from the project id and file id. - * - * @param {Map} entities - * @param {string} projectId - * @returns {Promise} - */ -async function readDeletedFiles(entities, projectId) { - const deletedFilesCursor = DeletedFile.find( - { - projectId: ObjectId(projectId), - }, - // only read the fields we need to save memory - { _id: 1, name: 1, deletedAt: 1 } - ) - .lean() - .cursor() - - for await (const file of deletedFilesCursor) { - // skip malformed deleted file entries - if (!file?._id) { - logger.warn({ file }, 'skipping deleted file with missing id') - continue - } - const id = file._id.toString() - // TODO: check if it already exists? - if (!entities.has(id)) { - // const ts = Number( - // file.deletedAt ? new Date(file.deletedAt) : Date.now() - // ) - - // TODO: would the hash be useful here? - - if (file.name && !SafePath.isCleanFilename(file.name)) { - const newName = SafePath.clean(file.name) - logger.warn( - { projectId, fileId: id, origName: file.name, newName }, - 'renaming invalid deleted file' - ) - file.name = newName - } - - entities.set(id, { - // NOTE: adding the doc id to the file path to avoid collisions - path: `/_deleted/${id}/${file.name}`, - name: file.name, - deleted: true, - type: 'file', - deletedAt: file.deletedAt, - }) - } - } -} - -/** - * Iterate through the sorted array of updates, pushing each one to Redis. - * - * In batches, tell project-history to pull the updates from Redis and process them, - * so the process fails early if something can't be processed. - * - * @param {Array} updates - * @param {string} projectId - * @param {string} projectHistoryId - * @param {Map.} fileMap - * @returns {Promise} - */ -async function sendUpdatesToProjectHistory( - updates, - projectId, - projectHistoryId, - fileMap -) { - let multi = rclient.multi() - let counter = 0 - let processed = 0 - let size = 0 - - const projectHistoryKey = - settings.redis.project_history_migration.key_schema.projectHistoryOps({ - projectId, - }) - - // clear out anything in the Redis queue for this project's history - multi.del(projectHistoryKey) - - for (let update of updates) { - // read the content for each update stub from the archive - if (update.stub) { - update = await buildEditDocUpdate(projectHistoryId, update, fileMap) - } - - // non-edit doc updates need string timestamps, not numbers - if (!('op' in update)) { - update.meta.ts = new Date(update.meta.ts).toISOString() - } - - const updateJSON = JSON.stringify(update) - multi.rpush(projectHistoryKey, updateJSON) - counter++ - processed++ - size += updateJSON.length - - // flush the history after every 1000 updates and start a new transaction - if (counter === 1000) { - logger.debug( - { processed, total: updates.length }, - 'sending updates to project history' - ) - // execute the transaction - await util.promisify(multi.exec)() - // tell project-history to pull the updates from the Redis queue - await HistoryManager.promises.flushProject(projectId) // TODO: roll back if this fails? - counter = 0 - size = 0 - multi = rclient.multi() - } else if (size > 1024 * 1024) { - // queue entries in redis more frequently to reduce memory usage - await util.promisify(multi.exec)() - size = 0 - multi = rclient.multi() - } - } - - if (counter > 0) { - // execute the transaction - await util.promisify(multi.exec)() - // tell project-history to pull the updates from the Redis queue - await HistoryManager.promises.flushProject(projectId) // TODO: roll back if this fails? - } - - // return the queue length so we can check that it is empty - const queueLength = await rclient.llen(projectHistoryKey) - return queueLength -} - -/** - * Compare two arrays of updates, with the earliest timestamp at the end first. - * - * @param {Array} a - * @param {Array} b - * @returns {number} - */ -function earliestTimestampFirst(a, b) { - // both arrays are empty, leave them - if (!a.length && !b.length) { - return 0 - } - - // a is empty, move b before a - if (!a.length) { - return 1 - } - - // b is empty, don't move b before a - if (!b.length) { - return -1 - } - - const tsB = b[b.length - 1].meta.ts - const tsA = a[a.length - 1].meta.ts - // if the last item in b has a lower timestamp that the last item in a, move b above a - if (tsB < tsA) { - return 1 - } - if (tsB > tsA) { - return -1 - } - // use pathnames as secondary sort key, to make order deterministic for - // updates with the same timestamp - const pathnameB = b[b.length - 1].pathname - const pathnameA = a[a.length - 1].pathname - if (pathnameB < pathnameA) { - return 1 - } - if (pathnameB > pathnameA) { - return -1 - } - return 0 // shouldn't happen, because pathnames must be distinct -} - -/** - * Compare two updates, with the highest version number first - * - * @param {AnyUpdate} a - * @param {AnyUpdate} b - * @returns {number} - */ -function decreasingDocVersion(a, b) { - if (b.v === a.v) { - throw new Error(`Matching version: ${b.v} ${a.v}`) - // return 0 - } - // if b.v is greater than a.v, sort b above a - return b.v > a.v ? 1 : -1 -} - -/** - * Create an array of queued updates for each doc/file, sorted by version - * - * @param {Array} updates - * @returns {Promise>} - */ -async function sortUpdatesByQueue(updates) { - // build a queue of updates for each doc/file - const queues = {} - - for (const update of updates) { - const docId = update.doc || update.file - - if (!(docId in queues)) { - queues[docId] = [] - } - - queues[docId].push(update) - } - - // convert the map to an array of queues - const values = Object.values(queues) - - for (const queue of values) { - // sort each queue in place, with each update in decreasing version ofder - queue.sort(decreasingDocVersion) - } - - return values -} - -/** - * Fetch all the content and updates for this project from track-changes, as a zip archive. - * - * @param {string} projectId - * @param {string} tempFilePath - * @returns - */ -async function fetchTrackChangesArchive(projectId, tempFilePath) { - const writeStream = fs.createWriteStream(tempFilePath) - - const url = `${settings.apis.trackchanges.url}/project/${projectId}/zip` - - // exposed for debugging during full-project-history migration - const timeout = - parseInt(process.env.FETCH_TRACK_CHANGES_TIMEOUT, 10) || 2 * 60 * 1000 - - try { - await util.promisify(pipeline)(request(url, { timeout }), writeStream) - } catch (err) { - logger.error({ err }, 'Error fetching track changes archive') - throw err - } - - const { size } = await fs.promises.stat(tempFilePath) - logger.info({ projectId, size }, 'fetched zip file from track-changes') -} - -/** - * Open the zip archive and build a Map of each entry in the archive, with the path as the key - * - * @param {string} filePath - * @returns {Promise>} - */ - -async function openTrackChangesArchive(filePath) { - const directory = await unzipper.Open.file(filePath) - return new Map(directory.files.map(file => [file.path, file])) -} - -/** - * Read the manifest data from the zip archive - * - * @param {Map} fileMap - * @returns {Promise} - */ -async function readTrackChangesManifest(fileMap) { - const manifestBuffer = await fileMap.get('manifest.json').buffer() - - return JSON.parse(manifestBuffer.toString()) -} - -/** - * Check that entities conform to the pathnames allowed by project history - * - * @param {Map} entities - * @param {string} projectId - */ -function validatePaths(entities, projectId) { - const pathErrors = [] - for (const [id, entity] of entities) { - if (!SafePath.isCleanPath(entity.path)) { - pathErrors.push( - `${entity.type}:${id}${entity.deleted ? ' (deleted)' : ''} path:${ - entity.path - }` - ) - } - } - if (pathErrors.length) { - throw new OError('Invalid path in history migration', { - projectId, - pathErrors, - }) - } -} - -/** - * Build an "add" update for an entity, with docLines or url set for the content. - * This represents a doc or file being added to a project. - * - * @param {Object} entity - * @param {string} entityId - * @param {string} projectId - * @param {string} projectHistoryId - * - * @returns {AddDocUpdate | AddFileUpdate} - */ -function buildAddUpdate(entity, entityId, projectId, projectHistoryId) { - const ts = new ObjectId(entityId).getTimestamp() - - const update = { - pathname: entity.path, - v: 0, // NOTE: only for sorting - meta: { - // source? - user_id: null, // TODO: assign the update to a system user? - ts: Number(ts), - origin: { kind: 'history-migration' }, - }, - projectHistoryId, - } - - switch (entity.type) { - case 'doc': { - return { - doc: entityId, - ...update, - docLines: entity.docLines, - } - } - - case 'file': { - // TODO: set a hash here? - return { - // type: 'external', - file: entityId, - ...update, - url: FilestoreHandler._buildUrl(projectId, entityId), - } - } - - default: - throw new Error('Unknown entity type') - } -} - -/** - * Build a "delete" update for an entity, with new_pathname set to an empty string. - * This represents a doc or file being deleted from a project. - * - * @param {Object} entity - * @param {string} entityId - * @param {string} projectId - * @param {string} projectHistoryId - * @returns DeleteUpdate - */ -function buildDeleteUpdate(entity, entityId, projectId, projectHistoryId) { - const ts = entity.deletedAt || new Date() - - const update = { - pathname: entity.path, - new_pathname: '', // empty path = deletion - v: Infinity, // NOTE: only for sorting - meta: { - user_id: null, // TODO: assign this to a system user? - ts: Number(ts), - origin: { kind: 'history-migration' }, - }, - projectHistoryId, - } - - switch (entity.type) { - case 'doc': - return { - doc: entityId, - ...update, - } - - case 'file': - return { - file: entityId, - ...update, - } - - default: - throw new Error(`Unknown entity type ${entity.type}`) - } -} - -/** - * @typedef TrackedDocUpdateMeta - * @property {string} user_id - * @property {number} start_ts - */ - -/** - * @typedef TrackedDocUpdate - * @property {string} doc_id - * @property {Array} op - * @property {number} v - * @property {TrackedDocUpdateMeta} meta - */ - -/** - * Build an "edit" update, with op set to an array of operations from track-changes. - * - * This represents the contents of a doc being edited in a project. - * - * @param {string} projectHistoryId - * @param {EditDocUpdateStub} updateStub - * @param {Map.} fileMap - * - * @returns {Promise} - */ -async function buildEditDocUpdate(projectHistoryId, updateStub, fileMap) { - const buffer = await fileMap.get(updateStub.path).buffer() - - /** - * @type TrackedDocUpdate - */ - const data = JSON.parse(buffer.toString()) - let userId = data.meta.user_id - if (userId === 'anonymous-user' || userId === 'null') { - userId = null - } - if (userId != null && !/^[0-9a-f]{24}$/.test(userId)) { - throw new OError('Bad user id in ShareLaTeX history edit update', { - userId, - }) - } - - return { - doc: data.doc_id, - op: data.op, // NOTE: this is an array of operations - v: data.v, - lastV: data.v - 1, - meta: { - user_id: userId, - ts: data.meta.start_ts, // TODO: use data.meta.end_ts or update.ts? - pathname: updateStub.pathname, - doc_length: updateStub.doc_length, - origin: { kind: 'history-migration' }, - }, - projectHistoryId, - } -} - -/** - * Build a stub for an "edit" update, with all the metadata but not the actual operations. - * - * This represents a doc being edited in a project, with enough information for sorting, - * but avoids loading the actual operations from the zip archive until they're needed, - * so as not to run out of memory if the project's history is large. - * - * @param {ManifestUpdate} update - * @param {Entity} entity - * @param {string} docId - * @returns {EditDocUpdateStub} - */ -function buildEditUpdateStub(update, entity, docId) { - return { - stub: true, - doc: docId, - v: update.version, - path: update.path, - pathname: entity.path, - doc_length: update.doc_length, - meta: { - ts: update.ts, - origin: { kind: 'history-migration' }, - }, - } -} - -/** - * Build the sorted array of updates to be sent to project-history. - * - * 1. Process all the added and edited files from the track-changes archive. - * 2. Process the other files from the project that have been added, and maybe deleted, without any edits. - * - * @param {string} projectId - * @param {string} projectHistoryId - * @param {Manifest} manifest - * @param {Map.} entities - * @param {Map.} fileMap - * @returns {Promise>} - */ -async function buildUpdates( - projectId, - projectHistoryId, - manifest, - entities, - fileMap -) { - /** - * @type Array - */ - const updates = [] - - // keep a list of doc ids which have updates in track-changes - const updatedDocs = new Set() - - // process the existing docs with updates, from track-changes - for (const doc of manifest.docs) { - const entity = entities.get(doc.id) - - if (!entity) { - throw new Error(`Entity not found for ${doc.id}`) - } - - if (!entity.path) { - throw new Error(`Path not found for ${doc.id}`) - } - - // add the initial content - const contentStart = doc.content.start - - const buffer = await fileMap.get(contentStart.path).buffer() - - /** - * @type AddDocUpdate - */ - const update = { - doc: doc.id, - pathname: entity.path, - v: contentStart.version - 1, - meta: { - user_id: null, // TODO: assign this to a system user? - ts: Number(ObjectId(doc.id).getTimestamp()), - origin: { kind: 'history-migration' }, - }, - projectHistoryId, - docLines: buffer.toString(), - } - - updates.push(update) - - // push the update onto the array of updates - for (const update of doc.updates) { - updates.push(buildEditUpdateStub(update, entity, doc.id)) - } - - updatedDocs.add(doc.id) - } - - // process the docs which have been added/deleted without any updates being recorded - for (const [id, entity] of entities.entries()) { - if (entity.deleted) { - // deleted entity - - // add the doc/file - if (!updatedDocs.has(id)) { - updates.push(buildAddUpdate(entity, id, projectId, projectHistoryId)) - } - - // delete the doc/file again (there may be updates added between adding and deleting) - updates.push(buildDeleteUpdate(entity, id, projectId, projectHistoryId)) - } else { - if (!updatedDocs.has(id)) { - // add "not deleted" doc that isn't in the manifest either - updates.push(buildAddUpdate(entity, id, projectId, projectHistoryId)) - } - } - } - - return updates -} - -/** - * Remove the `overleaf.history` object from the project and tell project-history to delete everything for this project. - * (note: project-history may not delete the actual history data yet, but it will at least delete the cached history id) - * - * @param {string} projectId - * @returns {Promise} - */ -async function deleteProjectHistory(projectId) { - // look up the history id from the project - const historyId = await ProjectHistoryHandler.promises.getHistoryId(projectId) - // delete the history from project-history and history-v1 - await HistoryManager.promises.deleteProject(projectId, historyId) - // TODO: send a message to document-updater? - await ProjectHistoryHandler.promises.unsetHistory(projectId) -} - -/** - * Send the updates from the track changes zip file to project history - * - * @param {string} projectId - * @param {string} projectHistoryId - * @param {Array} updates - * @param {Map.} fileMap - */ -async function migrateTrackChangesUpdates( - projectId, - projectHistoryId, - updates, - fileMap -) { - // Build a queue for each doc, sorted by version (and by timestamp within each version) - const queues = await sortUpdatesByQueue(updates) - - const sortedUpdates = [] - - let item - do { - // Find the earliest item from the tail of all queues - queues.sort(earliestTimestampFirst) - item = queues[0].pop() - if (item) { - sortedUpdates.push(item) - } - } while (item) - - // NOTE: leaving the version string code commented out, in case it ends up being needed - // let majorVersion = 0 - // let minorVersion = 0 - for (const update of sortedUpdates) { - // increment majorVersion if this is a file change - if (!('op' in update)) { - // remove v (only used for sorting) - delete update.v - - // set version - // majorVersion++ - // // minorVersion = 0 - // update.version = `${majorVersion}.${minorVersion}` // NOTE: not set as project-history doesn't need it and could cause problems if it gets higher than project.version - } - // increment minorVersion after every update - // minorVersion++ - } - - // add each update to the Redis queue for project-history to process - logger.debug( - { projectId, projectHistoryId }, - 'Sending updates for project to Redis' - ) - - const remainingQueueLength = await sendUpdatesToProjectHistory( - sortedUpdates, - projectId, - projectHistoryId, - fileMap - ) - // Failure will cause queued updates to be deleted (in the catch below) - - logger.debug( - { - projectId, - projectHistoryId, - remainingQueueLength, - }, - 'Updates sent to project-history' - ) - - if (remainingQueueLength > 0) { - throw new Error('flush to project-history did not complete') - } - - // TODO: roll back if any of the following fail? - - // TODO: check that the Redis queue is empty? - - // Clear any old entries in the main project history queue (these will not - // have a history id) - await HistoryManager.promises.flushProject(projectId) -} - -/** - * Add the zip file from track changes to the project file tree. - * We may be able to recover a failed history from the zip file in future. - * - * @param {string} projectId - * @param {string} rootFolderId - * @param {string} tempFilePath - */ - -async function uploadTrackChangesArchiveToProject( - projectId, - rootFolderId, - tempFilePath -) { - const { size } = await fs.promises.stat(tempFilePath) - if (size > settings.maxUploadSize) { - throw new FileTooLargeError({ - message: 'track-changes archive exceeds maximum size for archiving', - info: { size }, - }) - } - const { fileRef } = await ProjectEntityUpdateHandler.promises.addFile( - projectId, - rootFolderId, // project.rootFolder[0]._id, - `OverleafHistory-${new Date().toISOString().substring(0, 10)}.zip`, - tempFilePath, - null, - null, // no owner - null // no source - ) - logger.debug( - { projectId, fileRef }, - 'Uploaded track-changes zip archive to project due to error in migration' - ) -} - -/** - * Check all updates for invalid characters (nonBMP or null) and substitute - * the unicode replacement character if options.fixInvalidCharacters is true, - * otherwise throw an exception. - * @param {Array} updates - * @param {string} projectId - * @param {Object} options - */ -function validateUpdates(updates, projectId, options) { - const replace = options.fixInvalidCharacters - // check for invalid characters - function containsBadChars(str) { - return /[\uD800-\uDBFF]/.test(str) || str.indexOf('\x00') !== -1 - } - // Replace invalid characters so that they will be accepted by history_v1. - function sanitise(str) { - if (replace) { - return str.replace(/[\uD800-\uDFFF]/g, '\uFFFD').replace('\x00', '\uFFFD') - } else { - throw new Error('invalid character in content') - } - } - // Check size of doclines in update against max size allowed by history_v1. - // This catches docs which are too large when created, but not when they - // go over the limit due to edits. - function checkSize(update) { - if (update?.docLines?.length > settings.max_doc_length) { - throw new FileTooLargeError({ - message: 'docLines exceeds maximum size for history', - info: { docId: update.doc, size: update.docLines.length }, - }) - } - } - let latestTimestamp = 0 - // Iterate over the all the updates and their doclines or ops - for (const update of updates) { - checkSize(update) - // Find the timestamp of the most recent edit (either adding a doc or editing a doc) - // we exclude deletions as these are created in the migration and we didn't record - // the deletion time for older files. - const isDeleteUpdate = update.new_pathname === '' - if ( - update.doc && - !isDeleteUpdate && - update.meta.ts && - update.meta.ts > latestTimestamp - ) { - latestTimestamp = update.meta.ts - } - if (update.docLines && containsBadChars(update.docLines)) { - logger.debug({ update, replace }, 'invalid character in docLines') - update.docLines = sanitise(update.docLines) - } - if (update.op) { - for (const op of update.op) { - if (op.i && containsBadChars(op.i)) { - logger.debug({ update, replace }, 'invalid character in insert op') - op.i = sanitise(op.i) - } - if (op.d && containsBadChars(op.d)) { - logger.debug({ update, replace }, 'invalid character in delete op') - op.d = sanitise(op.d) - } - } - } - } - logger.debug( - { projectId, latestTimestamp, date: new Date(latestTimestamp) }, - 'timestamp of most recent edit' - ) - if (options.cutoffDate && new Date(latestTimestamp) > options.cutoffDate) { - throw new Error('project was edited after cutoff date') - } -} - -/** - * Migrate a project's history from track-changes to project-history - * - * @param {string} projectId - * - * @returns {Promise} - */ -async function migrateProjectHistory(projectId, options = {}) { - await fse.ensureDir(settings.path.projectHistories) - const projectHistoriesDir = await fs.promises.realpath( - settings.path.projectHistories - ) - const tempDir = await fs.promises.mkdtemp(projectHistoriesDir + path.sep) - const tempFilePath = path.join(tempDir, 'project.zip') - - try { - // fetch the zip archive of rewound content and updates from track-changes - // store the zip archive to disk, open it and build a Map of the entries - if (options.importZipFilePath) { - // use an existing track-changes archive on disk - logger.debug( - { src: options.importZipFilePath, dst: tempFilePath }, - 'importing zip file' - ) - await fs.promises.copyFile(options.importZipFilePath, tempFilePath) - const { size } = await fs.promises.stat(tempFilePath) - logger.info({ projectId, size }, 'imported zip file from disk') - } else { - await fetchTrackChangesArchive(projectId, tempFilePath) - } - const fileMap = await openTrackChangesArchive(tempFilePath) - - // read the manifest from the zip archive - const manifest = await readTrackChangesManifest(fileMap) - - // check that the project id in the manifest matches - // to be sure we are using the correct zip file - if (manifest.projectId !== projectId) { - throw new Error(`Incorrect projectId: ${manifest.projectId}`) - } - - // load the Project from MongoDB - const project = await ProjectGetter.promises.getProject(projectId) - - // create a history id for this project - const oldProjectHistoryId = _.get(project, 'overleaf.history.id') - - // throw an error if there is already a history associated with the project - if (oldProjectHistoryId) { - throw new Error( - `Project ${projectId} already has history ${oldProjectHistoryId}` - ) - } - - try { - // initialize a new project history and use the history id - // NOTE: not setting the history id on the project yet - const projectHistoryId = await HistoryManager.promises.initializeProject( - projectId - ) - - try { - // build a Map of the entities (docs and fileRefs) currently in the project, - // with _id as the key - const entities = await processRootFolder(project) - - // find all the deleted docs for this project and add them to the entity map - await readDeletedDocs(entities, projectId) - - // find all the deleted files for this project and add them to the entity map - await readDeletedFiles(entities, projectId) - - // check that the paths will not be rejected - validatePaths(entities, projectId) - - // build the array of updates that make up the new history for this project - const updates = await buildUpdates( - projectId, - projectHistoryId, - manifest, - entities, - fileMap - ) - - // check that the updates don't contain any characters that will be rejected by history_v1. - validateUpdates(updates, projectId, options) - - if (updates.length) { - await migrateTrackChangesUpdates( - projectId, - projectHistoryId, - updates, - fileMap - ) - } - } catch (error) { - if (options?.archiveOnFailure) { - // on error, optionally store the zip file in the project for future reference - logger.debug( - { projectId, error }, - 'Error sending track-changes updates to project history, attempting to archive zip file in project' - ) - try { - await uploadTrackChangesArchiveToProject( - projectId, - project.rootFolder[0]._id, - tempFilePath - ) - } catch (error) { - if (error instanceof InvalidNameError) { - logger.info({ projectId }, 'zip file already archived in project') - } else { - throw error - } - } finally { - // roll back the last updated timestamp and user - logger.debug( - { projectId }, - 'rolling back last updated time after uploading zip file' - ) - await ProjectUpdateHandler.promises.resetUpdated( - projectId, - project.lastUpdated, - project.lastUpdatedBy - ) - } - // set the overleaf.history.zipFileArchivedInProject flag for future reference - await ProjectHistoryHandler.promises.setMigrationArchiveFlag( - projectId - ) - // we consider archiving the zip file as "success" (at least we've given up on attempting - // to migrate the history) so we don't rethrow the error and continue to initialise the new - // empty history below. - } else { - // if we're not archiving the zip file then we rethrown the error to fail the migration - throw error - } - } - - // set the project's history id once the updates have been successfully processed - // (or we have given up and archived the zip file in the project). - logger.debug( - { projectId, projectHistoryId }, - 'Setting history id on project' - ) - await ProjectHistoryHandler.promises.setHistoryId( - projectId, - projectHistoryId - ) - - try { - // tell document updater to reload docs with the new history id - logger.debug({ projectId }, 'Asking document-updater to clear project') - await DocumentUpdaterHandler.promises.flushProjectToMongoAndDelete( - projectId - ) - - // run a project history resync in case any changes have arrived since the migration - logger.debug( - { projectId }, - 'Asking project-history to force resync project' - ) - - await HistoryManager.promises.resyncProject(projectId, { - force: true, - origin: { kind: 'history-migration' }, - }) - } catch (error) { - if (options.forceNewHistoryOnFailure) { - logger.warn( - { projectId }, - 'failed to resync project, forcing new history' - ) - } else { - throw error - } - } - logger.debug( - { projectId }, - 'Switching on full project history display for project' - ) - // Set the display to v2 history but allow downgrading (second argument allowDowngrade = true) - await ProjectHistoryHandler.promises.upgradeHistory(projectId, true) - } catch (error) { - // delete the history id again if something failed? - logger.warn( - OError.tag( - error, - 'Something went wrong flushing and resyncing project; clearing full project history for project', - { projectId } - ) - ) - await deleteProjectHistory(projectId) - - throw error - } - } finally { - // clean up the temporary directory - await fse.remove(tempDir) - } -} diff --git a/services/web/modules/history-migration/index.js b/services/web/modules/history-migration/index.js deleted file mode 100644 index 4ba52ba2c8..0000000000 --- a/services/web/modules/history-migration/index.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = {} diff --git a/services/web/modules/history-migration/test/unit/src/ProjectHistoryControllerTests.js b/services/web/modules/history-migration/test/unit/src/ProjectHistoryControllerTests.js deleted file mode 100644 index 52d1bf9f42..0000000000 --- a/services/web/modules/history-migration/test/unit/src/ProjectHistoryControllerTests.js +++ /dev/null @@ -1,346 +0,0 @@ -const sinon = require('sinon') -const nock = require('nock') -const { expect } = require('chai') -const fs = require('fs') -const path = require('path') -const SandboxedModule = require('sandboxed-module') -const { ObjectId } = require('mongodb') -const unzipper = require('unzipper') - -const modulePath = '../../../app/src/ProjectHistoryController' - -describe('ProjectHistoryController', function () { - const projectId = ObjectId('611bd20c5d76a3c1bd0c7c13') - const deletedFileId = ObjectId('60f6e92c6c14d84fb7a71ae1') - const historyId = 123 - - let clock - const now = new Date(Date.UTC(2021, 1, 1, 0, 0)).getTime() - - before(async function () { - clock = sinon.useFakeTimers({ - now, - shouldAdvanceTime: true, - }) - }) - - after(function () { - // clock.runAll() - clock.restore() - }) - - beforeEach(function () { - this.db = { - users: { - countDocuments: sinon.stub().yields(), - }, - } - - this.project = { - _id: ObjectId('611bd20c5d76a3c1bd0c7c13'), - name: 'My Test Project', - rootDoc_id: ObjectId('611bd20c5d76a3c1bd0c7c15'), - rootFolder: [ - { - _id: ObjectId('611bd20c5d76a3c1bd0c7c12'), - name: 'rootFolder', - folders: [ - { - _id: ObjectId('611bd242e64281c13303d6b5'), - name: 'a folder', - folders: [ - { - _id: ObjectId('611bd247e64281c13303d6b7'), - name: 'a subfolder', - folders: [], - fileRefs: [], - docs: [ - { - _id: ObjectId('611bd24ee64281c13303d6b9'), - name: 'a renamed file in a subfolder.tex', - }, - ], - }, - ], - fileRefs: [], - docs: [], - }, - { - _id: ObjectId('611bd34ee64281c13303d6be'), - name: 'images', - folders: [], - fileRefs: [ - { - _id: ObjectId('611bd2bce64281c13303d6bb'), - name: 'overleaf-white.svg', - linkedFileData: { - provider: 'url', - url: 'https://cdn.overleaf.com/img/ol-brand/overleaf-white.svg', - }, - created: '2021-08-17T15:16:12.753Z', - }, - ], - docs: [], - }, - ], - fileRefs: [ - { - _id: ObjectId('611bd20c5d76a3c1bd0c7c19'), - name: 'universe.jpg', - linkedFileData: null, - created: '2021-08-17T15:13:16.400Z', - }, - ], - docs: [ - { - _id: ObjectId('611bd20c5d76a3c1bd0c7c15'), - name: 'main.tex', - }, - { - _id: ObjectId('611bd20c5d76a3c1bd0c7c17'), - name: 'references.bib', - }, - ], - }, - ], - compiler: 'pdflatex', - description: '', - deletedDocs: [], - members: [], - invites: [], - owner: { - _id: ObjectId('611572e24bff88527f61dccd'), - first_name: 'Test', - last_name: 'User', - email: 'test@example.com', - privileges: 'owner', - signUpDate: '2021-08-12T19:13:38.462Z', - }, - features: {}, - } - - this.multi = { - del: sinon.stub(), - rpush: sinon.stub(), - exec: sinon.stub().yields(null, 1), - } - - const { docs, folders } = this.project.rootFolder[0] - - const allDocs = [...docs] - - const processFolders = folders => { - for (const folder of folders) { - for (const doc of folder.docs) { - allDocs.push(doc) - } - - if (folder.folders) { - processFolders(folder.folders) - } - } - } - - processFolders(folders) - - allDocs.forEach(doc => { - doc.lines = [`this is the contents of ${doc.name}`] - }) - - // handle Doc.find().lean().cursor() - this.findDocs = sinon.stub().returns({ - lean: sinon.stub().returns({ - cursor: sinon.stub().returns(allDocs), - }), - }) - - // handle await Doc.findOne().lean() - single result, no cursor required - this.findOneDoc = sinon.stub().callsFake(id => { - const result = allDocs.find(doc => { - return doc._id.toString() === id.toString() - }) - return { lean: sinon.stub().resolves(result) } - }) - - this.deletedFiles = [ - { - _id: deletedFileId, - name: 'testing.tex', - deletedAt: new Date(), - }, - ] - - // handle DeletedFile.find().lean().cursor() - this.findDeletedFiles = sinon.stub().returns({ - lean: sinon - .stub() - .returns({ cursor: sinon.stub().returns(this.deletedFiles) }), - }) - - this.ProjectGetter = { - promises: { - getProject: sinon.stub().resolves(this.project), - }, - } - - this.FileStoreHandler = { - _buildUrl: (projectId, fileId) => - `http://filestore.test/${projectId}/${fileId}`, - } - - this.ProjectHistoryHandler = { - promises: { - setHistoryId: sinon.stub(), - upgradeHistory: sinon.stub(), - }, - } - - this.ProjectEntityUpdateHandler = { - promises: { - resyncProjectHistory: sinon.stub(), - }, - } - - this.DocumentUpdaterHandler = { - promises: { - flushProjectToMongoAndDelete: sinon.stub(), - }, - } - - this.HistoryManager = { - promises: { - resyncProject: sinon.stub(), - flushProject: sinon.stub(), - initializeProject: sinon.stub().resolves(historyId), - }, - } - - this.settings = { - redis: { - project_history_migration: { - key_schema: { - projectHistoryOps({ projectId }) { - return `ProjectHistory:Ops:{${projectId}}` // NOTE: the extra braces are intentional - }, - }, - }, - }, - apis: { - documentupdater: { - url: 'http://document-updater', - }, - trackchanges: { - url: 'http://track-changes', - }, - project_history: { - url: 'http://project-history', - }, - }, - path: { - projectHistories: 'data/projectHistories', - }, - } - - this.ProjectHistoryController = SandboxedModule.require(modulePath, { - requires: { - '../../../../app/src/Features/Project/ProjectGetter': - this.ProjectGetter, - '../../../../app/src/Features/FileStore/FileStoreHandler': - this.FileStoreHandler, - '../../../../app/src/Features/Project/ProjectHistoryHandler': - this.ProjectHistoryHandler, - '../../../../app/src/Features/Project/ProjectUpdateHandler': - this.ProjectUpdateHandler, - '../../../../app/src/Features/Project/ProjectEntityUpdateHandler': - this.ProjectEntityUpdateHandler, - '../../../../app/src/Features/History/HistoryManager': - this.HistoryManager, - '../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler': - this.DocumentUpdaterHandler, - '../../../../app/src/models/Doc': { - Doc: { - find: this.findDocs, - findOne: this.findOneDoc, - }, - }, - '../../../../app/src/models/DeletedFile': { - DeletedFile: { - find: this.findDeletedFiles, - }, - }, - '../../../../app/src/infrastructure/mongodb': { - db: this.db, - }, - '../../../../app/src/infrastructure/Mongoose': { - Schema: { - ObjectId: sinon.stub(), - Types: { - Mixed: sinon.stub(), - }, - }, - }, - '../../../../app/src/infrastructure/RedisWrapper': { - client: () => ({ - multi: () => this.multi, - llen: sinon.stub().resolves(0), - }), - }, - unzipper: { - Open: { - file: () => - unzipper.Open.file( - path.join(__dirname, 'data/track-changes-project.zip') - ), - }, - }, - '@overleaf/settings': this.settings, - }, - }) - }) - - afterEach(function () { - nock.cleanAll() - }) - - it('migrates a project history', async function () { - const readStream = fs.createReadStream( - path.join(__dirname, 'data/track-changes-project.zip') - ) - - nock(this.settings.apis.trackchanges.url) - .get(`/project/${projectId}/zip`) - .reply(200, readStream) - - nock(this.settings.apis.project_history.url) - .post(`/project`) - .reply(200, { project: { id: historyId } }) - - await this.ProjectHistoryController.migrateProjectHistory( - projectId.toString(), - 5 - ) - - expect(this.multi.exec).to.have.been.calledOnce - expect(this.ProjectHistoryHandler.promises.setHistoryId).to.have.been - .calledOnce - // expect(this.ProjectEntityUpdateHandler.promises.resyncProjectHistory).to - // .have.been.calledOnce - expect(this.HistoryManager.promises.flushProject).to.have.been.calledTwice - expect(this.multi.rpush).to.have.callCount(12) - - const args = this.multi.rpush.args - - const snapshotPath = path.join( - __dirname, - 'data/migrate-project-history.snapshot.json' - ) - - // const snapshot = JSON.stringify(args, null, 2) - // await fs.promises.writeFile(snapshotPath, snapshot) - - const json = await fs.promises.readFile(snapshotPath, 'utf-8') - const expected = JSON.parse(json) - - expect(args).to.deep.equal(expected) - }) -}) diff --git a/services/web/modules/history-migration/test/unit/src/data/migrate-project-history.snapshot.json b/services/web/modules/history-migration/test/unit/src/data/migrate-project-history.snapshot.json deleted file mode 100644 index a2b37de5ae..0000000000 --- a/services/web/modules/history-migration/test/unit/src/data/migrate-project-history.snapshot.json +++ /dev/null @@ -1,50 +0,0 @@ -[ - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"file\":\"60f6e92c6c14d84fb7a71ae1\",\"pathname\":\"/_deleted/60f6e92c6c14d84fb7a71ae1/testing.tex\",\"meta\":{\"user_id\":null,\"ts\":\"2021-07-20T15:18:04.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"url\":\"http://filestore.test/611bd20c5d76a3c1bd0c7c13/60f6e92c6c14d84fb7a71ae1\"}" - ], - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"file\":\"60f6e92c6c14d84fb7a71ae1\",\"pathname\":\"/_deleted/60f6e92c6c14d84fb7a71ae1/testing.tex\",\"new_pathname\":\"\",\"meta\":{\"user_id\":null,\"ts\":\"2021-02-01T00:00:00.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}" - ], - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"doc\":\"611bd20c5d76a3c1bd0c7c15\",\"pathname\":\"/main.tex\",\"meta\":{\"user_id\":null,\"ts\":\"2021-08-17T15:13:16.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"docLines\":\"\\\\documentclass{article}\\n\\\\usepackage[utf8]{inputenc}\\n\\n\\\\title{My Test Project}\\n\\\\author{alf.eaton+dev }\\n\\\\date{7 2021}\\n\\n\\\\usepackage{natbib}\\n\\\\usepackage{graphicx}\\n\\n\\\\begin{document}\\n\\n\\\\maketitle\\n\\n\\\\section{Introduction}\\nThere is a theory which states that if ever anyone discovers exactly what the Universe is for and why it is here, it will instantly disappear and be replaced by something even more bizarre and inexplicable.\\nThere is another theory which states that this has already happened.\\n\\n\\\\begin{figure}[h!]\\n\\\\centering\\n\\\\includegraphics[scale=1.7]{universe}\\n\\\\caption{The Universe}\\n\\\\label{fig:universe}\\n\\\\end{figure}\\n\\n\\\\section{Conclusion}\\n``I always thought something was fundamentally wrong with the universe'' \\\\citep{adams1995hitchhiker}\\n\\n\\\\bibliographystyle{plain}\\n\\\\bibliography{references}\\n\\\\end{document}\\n\"}" - ], - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"doc\":\"611bd20c5d76a3c1bd0c7c17\",\"pathname\":\"/references.bib\",\"meta\":{\"user_id\":null,\"ts\":\"2021-08-17T15:13:16.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"docLines\":\"this is the contents of references.bib\"}" - ], - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"file\":\"611bd20c5d76a3c1bd0c7c19\",\"pathname\":\"/universe.jpg\",\"meta\":{\"user_id\":null,\"ts\":\"2021-08-17T15:13:16.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"url\":\"http://filestore.test/611bd20c5d76a3c1bd0c7c13/611bd20c5d76a3c1bd0c7c19\"}" - ], - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"doc\":\"611bd20c5d76a3c1bd0c7c15\",\"op\":[{\"p\":487,\"i\":\"\\n\\nAdding some text here.\"}],\"v\":1,\"lastV\":0,\"meta\":{\"user_id\":\"611572e24bff88527f61dccd\",\"ts\":1629213228148,\"pathname\":\"/main.tex\",\"doc_length\":805,\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}" - ], - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"doc\":\"611bd20c5d76a3c1bd0c7c15\",\"op\":[{\"p\":678,\"d\":\" something\"}],\"v\":2,\"lastV\":1,\"meta\":{\"user_id\":\"611572e24bff88527f61dccd\",\"ts\":1629213235181,\"pathname\":\"/main.tex\",\"doc_length\":829,\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}" - ], - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"doc\":\"611bd20c5d76a3c1bd0c7c15\",\"op\":[{\"d\":\" \",\"p\":722},{\"i\":\"\\n\",\"p\":722}],\"v\":3,\"lastV\":2,\"meta\":{\"user_id\":\"611572e24bff88527f61dccd\",\"ts\":1629213239472,\"pathname\":\"/main.tex\",\"doc_length\":819,\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}" - ], - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"doc\":\"611bd20c5d76a3c1bd0c7c15\",\"op\":[{\"p\":750,\"i\":\"\\n\\nAdding some text after deleting some text.\"}],\"v\":7,\"lastV\":6,\"meta\":{\"user_id\":\"611572e24bff88527f61dccd\",\"ts\":1629213241498,\"pathname\":\"/main.tex\",\"doc_length\":819,\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}" - ], - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"doc\":\"611bd24ee64281c13303d6b9\",\"pathname\":\"/a folder/a subfolder/a renamed file in a subfolder.tex\",\"meta\":{\"user_id\":null,\"ts\":\"2021-08-17T15:14:22.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"docLines\":\"\"}" - ], - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"doc\":\"611bd24ee64281c13303d6b9\",\"op\":[{\"p\":0,\"i\":\"Adding some content to the file in the subfolder.\"}],\"v\":2,\"lastV\":1,\"meta\":{\"user_id\":\"611572e24bff88527f61dccd\",\"ts\":1629213266076,\"pathname\":\"/a folder/a subfolder/a renamed file in a subfolder.tex\",\"doc_length\":0,\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}" - ], - [ - "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}", - "{\"file\":\"611bd2bce64281c13303d6bb\",\"pathname\":\"/images/overleaf-white.svg\",\"meta\":{\"user_id\":null,\"ts\":\"2021-08-17T15:16:12.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"url\":\"http://filestore.test/611bd20c5d76a3c1bd0c7c13/611bd2bce64281c13303d6bb\"}" - ] -] diff --git a/services/web/modules/history-migration/test/unit/src/data/track-changes-project.zip b/services/web/modules/history-migration/test/unit/src/data/track-changes-project.zip deleted file mode 100644 index 4767f19310b94a2ad87f61ea9eeef79f3d4872a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3527 zcma);c|4SB8^=eCWoV8Z6$d#S*+vY**h@*4Awwjbnv9)h3WI3qAX1i)CA*2na5(m| zglMD**%CFjY&A(zgUIqedf(2d*Xi^;pZoqi&p+S${@wR={jLj*V&@P9fw;Lret|oV zmAv2Fzg_ZGzbJjDl221RN#(kPU=Vpgd^|+cT{&&R#S33izhha2};g*CnXJE zOYO6CIREfGY(P$*Hy4V|lj|ZqOeE=X`Im2>IGja|V&v*&VgJ0ee&Jf-kcOT%gHST- zHCXM|s#`t)(%fxy2sv8tELxbWRQr{euE}D6edJ|}xS0ZXG3G9G`o($k#Si%W&PRc zj-pFiH&1HegH`&P@0LdSHoJSNNU0qo?SJx7PeKUWGOG0#uY(Z(8uN~u0F+I^Lnl>q zQ8uf(M0DUyZV!&JTWa;GA65q%qEN}U+s0c%p2WOpQF}}{#4qB~sB}ycd#=4|hx$wT z)8WWa!_-uzHLXF51JXLht97`FG7mLr+*u?Q84c}}p-rEW810`N!`nZoP!81Ly-&Q{ zmxM`!c3~-hHvT>@S;zal4=j!aKl-qmm9@vZiH$4|i>Zy9qx=i2L9xV=6uN5eC_+;n+jJ|d5A z#LA@kXuVU^lEUnK%^D~A?Nj5b8B=$Hi+5&9__m)2(e=JH&7Lo6 z81CY}cJ^lnCUE^b!_^vm(y&rzAhW9(+hXP}-*)P8qt(T=4CQqj;z;n0@yy(nNtH7g zmc)8Mq9v*Uautv^^rI3HfJA8MhD5B(5P0?0KBl|}_Xc?wI zXFPp+EH6-~m0UJqm>+Ii3cJf69jdTm%tg9de7;{q5nbkryJk4;9hI@>$N5QwrKAo} zGJWHaToj<9_eYhe1jW=to8kOn7;wPjuDS&A_wwAYnl~=R(nT!Km@eh_QnM%#xo!g+NR+yY4)m`O7_1h)(b`TQ zwW)l$UDMN(!=>_7GG22>M_X0oT9d(;zdT;_;~%S%??zQ~hTcVsE~mafhjp%WO%zf<6nHi5$!mc8_;;v@o%D7<@{YC(a);Z&XOwF05;%n9?IG zlU{Txtg!-h(SPT{Pb_g8)@w4`XHEmGcQ0VQvVg#UwO${BgEv7*+1IixmJa6|THX1Y zpdXc^nNxi#dwo4*&3}lF^Y&k;WyV5}2`F{4+U|x!d)|Hz^4Xuk$YTfeL|2E$m z_x$MJ26XwVGxOH(!l>g?4;Kbii{m&HMqDeJ6s8QXHg;V_Hc9O#ZzEwOj3ySLnCK}n z?qn{X^||1abd2k6P6p*dm`wynuFQqPTVhb^q)E98w6qnB880dotkEgUdqtk~#v}Lq z{L3TG==0I;NJFugX1i|1t8rl|>K{XCpw9SijYr7Wx*|FuNoe~@6MBa^dAXGDjpNL@ zWet&pv;wra2QiPIuMB$**B=6p<-Rzrt;*91+NG9d7Rx1Qu44r~1GQlIF+~!xsxpMk z9e?FLd9SZSOEd|+yl$xlnD24Gd|f`9@9MGgdB!^{44B;U&D$*3H)bkQ| zoB!Q(;Cv~p*8rqOYp#A(U(;>S%cjNW%S=hPbMiN+^LlMxq!%#9SXwep-Q(~Vb6oX& zjl~qjllikYr9qz(n|I=aRPfv+#Mj@|4i)Oiw(s$DYpZz{HPHNmY1{6bis;{w=@sqE zUVS{4E9y#n9ZIw+T8I+s%ZxS9*Wp0qzZ0nhgn=?JSe01S*%_&-qM_`lj6lE63zMDoRu-r#ruxx#cCBN9Zigiqz$9*8ZE*iAkap<0*bt{fon3VdK*dC4&(gVqo1NHk;tngGi zPiq}DlOA1~dpp3h|M-qdgllfG*U^fKYs00&$%$ll)%}`g z$^y_UyF=r11TJ=mx8HliiC|EQt&<16BdCrrj3^y*v?cy-I_{0W!HAdWCQI4KIzTFD z+r|VutgPEY^OuI_zgefkzUk~XILxqfYl=Yjig1EJ+W{8N0q^eO>_brW@HvaGPi(@6 z!TBO)b-<4T1iS9to13&Z(42bF1Qo9Jsx1Ue+Pe=2JfV z+~0L2cW-)z(I3UmPZE#Q(nN@)Bf_H-4%}764CU86odLnz=2rd%?J7w1uJcRh7H8Yj_&m`omarhvPN3HVK@pVqGLNw(@eHtV9-8Py$X9f>{QdZD|g$82vl-dmsN;2M4O#Is1oQ8PG zCG0x26-!iTvI&jg5U=|UL%cgiGz~MFPf51PUCPMpR(yNs+)+qTbR);nnk2f1d*f2f zr=S@X`s_#5;-T3~FFJ!;xh8@?)q}j+h$hBK(Tmv+YO}ZJkE^3mY}*d7|8r0J`L2Me zGGycHUv2Vphq{@$RR_ajN(0gGkIXND|J5G1Y8O~M9>C83m5128zpaWki`V-jJeAG7 zttuyrm&Ne|`%~V`+p1Wxco)Fm;eDM%|404S^H|iYTwkc)NzIL${^}&RGBp-+^?S@O xH~rN~ZeN2?D2p0@$GqjYacACCeJr!f<}P>j{$)oz$*>}Ljq*u?>{`$QpW%Q diff --git a/services/web/package.json b/services/web/package.json index 2fcb90d5e3..2e141c86fe 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -243,7 +243,6 @@ "scroll-into-view-if-needed": "^2.2.25", "tsscmp": "^1.0.6", "underscore": "^1.13.1", - "unzipper": "^0.10.11", "utf-8-validate": "^5.0.2", "uuid": "^3.0.1", "valid-data-url": "^2.0.0", diff --git a/services/web/scripts/fix_oversized_docs.js b/services/web/scripts/fix_oversized_docs.js index 3bdc073e12..3c7a69e302 100644 --- a/services/web/scripts/fix_oversized_docs.js +++ b/services/web/scripts/fix_oversized_docs.js @@ -147,7 +147,6 @@ async function deleteDocFromRedis(projectId, docId) { `UnflushedTime:{${docId}}`, `Pathname:{${docId}}`, `ProjectHistoryId:{${docId}}`, - `ProjectHistoryType:{${docId}}`, `PendingUpdates:{${docId}}`, `lastUpdatedAt:{${docId}}`, `lastUpdatedBy:{${docId}}` diff --git a/services/web/scripts/history/clean_sl_history_data.js b/services/web/scripts/history/clean_sl_history_data.js deleted file mode 100644 index 1800692cbe..0000000000 --- a/services/web/scripts/history/clean_sl_history_data.js +++ /dev/null @@ -1,60 +0,0 @@ -const { waitForDb, db } = require('../../app/src/infrastructure/mongodb') - -async function main() { - await checkAllProjectsAreMigrated() - await setAllowDowngradeToFalse() - await deleteHistoryCollections() - console.log('Legacy history data cleaned up successfully') - process.exit(0) -} - -async function checkAllProjectsAreMigrated() { - console.log('checking all projects are migrated to Full Project History') - - const count = await db.projects.countDocuments({ - 'overleaf.history.display': { $ne: true }, - }) - - if (count === 0) { - console.log('All projects are migrated to Full Project History') - } else { - console.error( - `There are ${count} projects that are not migrated to Full Project History` + - ` please complete the migration before running this script again.` - ) - process.exit(1) - } -} - -async function setAllowDowngradeToFalse() { - console.log('unsetting `allowDowngrade` flag in all projects') - await db.projects.updateMany( - { - 'overleaf.history.id': { $exists: true }, - 'overleaf.history.allowDowngrade': true, - }, - { $unset: { 'overleaf.history.allowDowngrade': 1 } } - ) - console.log('unsetting `allowDowngrade` flag in all projects - Done') -} - -async function deleteHistoryCollections() { - console.log('removing `docHistory` data') - await db.docHistory.deleteMany({}) - console.log('removing `docHistory` data - Done') - - console.log('removing `docHistoryIndex` data') - await db.docHistoryIndex.deleteMany({}) - console.log('removing `docHistoryIndex` data - Done') - - console.log('removing `projectHistoryMetaData` data') - await db.projectHistoryMetaData.deleteMany({}) - console.log('removing `projectHistoryMetaData` data - Done') -} - -waitForDb() - .then(main) - .catch(err => { - console.error(err) - process.exit(1) - }) diff --git a/services/web/scripts/history/count_project_history_categories.js b/services/web/scripts/history/count_project_history_categories.js deleted file mode 100644 index 0df4cfd6eb..0000000000 --- a/services/web/scripts/history/count_project_history_categories.js +++ /dev/null @@ -1,84 +0,0 @@ -const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true' -const VERBOSE_PROJECT_NAMES = process.env.VERBOSE_PROJECT_NAMES === 'true' -const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 50 -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 500 -const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false' -// persist fallback in order to keep batchedUpdate in-sync -process.env.BATCH_SIZE = BATCH_SIZE -// raise mongo timeout to 1hr if otherwise unspecified -process.env.MONGO_SOCKET_TIMEOUT = - parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 - -const { promiseMapWithLimit } = require('../../app/src/util/promises') -const { batchedUpdate } = require('../helpers/batchedUpdate') -const { - determineProjectHistoryType, - countProjects, -} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper') - -const COUNT = { - V2: 0, - V1WithoutConversion: 0, - V1WithConversion: 0, - NoneWithoutConversion: 0, - NoneWithConversion: 0, - NoneWithTemporaryHistory: 0, - UpgradeFailed: 0, - ConversionFailed: 0, - MigratedProjects: 0, - TotalProjects: 0, -} - -async function processBatch(projects) { - await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject) - console.log(COUNT) -} - -async function processProject(project) { - const historyType = await determineProjectHistoryType(project) - if (VERBOSE_LOGGING) { - console.log( - `project ${ - project[VERBOSE_PROJECT_NAMES ? 'name' : '_id'] - } is type ${historyType}` - ) - } - COUNT[historyType] += 1 -} - -async function main() { - const projection = { - _id: 1, - overleaf: 1, - } - const options = {} - if (USE_QUERY_HINT) { - options.hint = { _id: 1 } - } - if (VERBOSE_PROJECT_NAMES) { - projection.name = 1 - } - await batchedUpdate( - 'projects', - { 'overleaf.history.display': { $ne: true } }, - processBatch, - projection, - options - ) - COUNT.MigratedProjects = await countProjects({ - 'overleaf.history.display': true, - }) - COUNT.TotalProjects = await countProjects() - console.log('Final') - console.log(COUNT) -} - -main() - .then(() => { - console.error('Done.') - process.exit(0) - }) - .catch(error => { - console.error({ error }) - process.exit(1) - }) diff --git a/services/web/scripts/history/debug_history.js b/services/web/scripts/history/debug_history.js deleted file mode 100644 index 755716eb55..0000000000 --- a/services/web/scripts/history/debug_history.js +++ /dev/null @@ -1,328 +0,0 @@ -// Script to debug the track-changes history of the documents in a project. -// Usage: -// node debug_history.js --project-id= -// node debug_history.js --all # to check all unmigrated projects -// -// Example output: -// $ node scripts/debug_history.js --project-id=63ff3adc06177192f18a6b38 -// Using default settings from /overleaf/services/track-changes/config/settings.defaults.js -// Set UV_THREADPOOL_SIZE=16 -// project 63ff3adc06177192f18a6b38 docId 63ff3adc06177192f18a6b3d OK -// project 63ff3adc06177192f18a6b38 docId 63ff3b08de41e3b0989c1720 FAILED -// {"action":"rewinding","version":7,"meta":{"start_ts":1677671465447,"end_ts":1677671465447,"user_id":"632ae106f9a6dd002505765b"}, -// "ops":[{"action":"rewindOp","contentLength":24,"op":{"p":32,"d":6},"errors":[{"message":"invalid offset rewinding delete, -// truncating to content length","op":{"p":32,"d":6},"contentLength":24}]}],"status":"failed"} - -/* eslint-disable camelcase */ -const TrackChangesMongoDb = require('../../../track-changes/app/js/mongodb') -const { waitForDb } = require('../../app/src/infrastructure/mongodb') -const { - findProjects, -} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper') -const PackManager = require('../../../track-changes/app/js/PackManager') -const { - packsAreDuplicated, -} = require('../../../track-changes/app/js/util/PackUtils') -const { - ConsistencyError, -} = require('../../../track-changes/app/js/DiffGenerator') -const DocumentUpdaterManager = require('../../../track-changes/app/js/DocumentUpdaterManager') -const DocstoreManager = require('../../../track-changes/app/js/DocstoreManager') -const Errors = require('../../../track-changes/app/js/Errors') -const minimist = require('minimist') -const util = require('util') -const logger = require('@overleaf/logger') -logger.initialize('debug-history') -// disable logging to stdout from internal modules -logger.logger.streams = [] - -const options = { - boolean: ['all', 'verbose', 'raw', 'help'], - string: ['project-id'], - alias: { - 'project-id': 'p', - verbose: 'v', - raw: 'r', - help: 'h', - all: 'a', - }, - default: {}, -} -const argv = minimist(process.argv.slice(2), options) - -function usage() { - console.log( - `Usage: ${process.argv[1]} [--project-id= | --all] [--verbose] [--raw]` - ) - process.exit(1) -} - -// look in docstore or docupdater for the latest version of the document -async function getLatestContent(projectId, docId, lastUpdateVersion) { - const [docstoreContent, docstoreVersion] = - await DocstoreManager.promises.peekDocument(projectId, docId) - - // if docstore is out of date, check for a newer version in docupdater - // and return that instead - if (docstoreVersion <= lastUpdateVersion) { - const [docupdaterContent, docupdaterVersion] = - await DocumentUpdaterManager.promises.peekDocument(projectId, docId) - if (docupdaterVersion > docstoreVersion) { - return [docupdaterContent, docupdaterVersion] - } - } - - return [docstoreContent, docstoreVersion] -} - -// This class is used to write a record of all the operations that have been applied to a document -class LogAppliedOps { - constructor() { - this.result = [] - } - - // used to log the initial state of the document - start(action, latestContent, version) { - this.result.push({ - action, - latestContentLength: latestContent.length, - latestContent: argv.raw ? latestContent : undefined, - version, - }) - } - - // used to log a new document update - update(action, update) { - this._finalize() - this.opResults = [] - this.currentResult = { - action, - version: update.v, - meta: update.meta, - ops: this.opResults, - } - this.result.push(this.currentResult) - } - - // used to log an operation that has been applied to the document - op(action, content, op) { - this.currentOp = { - action, - contentLength: content.length, - content: argv.raw ? content : undefined, - op: this._filterOp(op), - } - this.opResults.push(this.currentOp) - } - - // used to log an error that occurred while applying an operation - opError(message, content, op, err) { - this.currentOp.errors = this.currentOp.errors || [] - this.currentOp.errors.push({ - message, - op: this._filterOp(op), - contentLength: content.length, - content: argv.raw ? content : undefined, - err, - }) - } - - // sets the status of the current update to 'success' or 'failed' - // depending on whether any errors were logged - _finalize() { - if (!this.currentResult) { - return - } - const errors = this.opResults.some(op => op.errors) - this.currentResult.status = errors ? 'failed' : 'success' - } - - // returns the final result of the log - end() { - this._finalize() - return this.result - } - - // Returns a new object with the same keys as op, but with the i and d - // fields replaced by their lengths when present. This is used to filter - // out the contents of the i and d fields of an operation, to redact - // document content. - _filterOp(op) { - const newOp = {} - for (const key of Object.keys(op)) { - if (!argv.raw && (key === 'i' || key === 'd')) { - newOp[key] = op[key].length - } else { - newOp[key] = op[key] - } - } - return newOp - } -} - -// This is the rewindOp function from track-changes, modified to log -// the operation and any errors. -function rewindOp(content, op, log) { - if (op.i != null) { - // ShareJS will accept an op where p > content.length when applied, - // and it applies as though p == content.length. However, the op is - // passed to us with the original p > content.length. Detect if that - // is the case with this op, and shift p back appropriately to match - // ShareJS if so. - let { p } = op - const maxP = content.length - op.i.length - if (p > maxP) { - log.opError( - 'invalid offset rewinding insert, truncating to content length', - content, - op - ) - p = maxP - } - const textToBeRemoved = content.slice(p, p + op.i.length) - if (op.i !== textToBeRemoved) { - log.opError( - 'inserted content does not match text to be removed', - content, - op - ) - throw new ConsistencyError( - `Inserted content, '${op.i}', does not match text to be removed, '${textToBeRemoved}'` - ) - } - return content.slice(0, p) + content.slice(p + op.i.length) - } else if (op.d != null) { - if (op.p > content.length) { - log.opError( - 'invalid offset rewinding delete, truncating to content length', - content, - op - ) - } - return content.slice(0, op.p) + op.d + content.slice(op.p) - } else { - return content - } -} - -// This is the rewindDoc function from track-changes, modified to log all -// operations that are applied to the document. -async function rewindDoc(projectId, docId) { - const log = new LogAppliedOps() - // Prepare to rewind content - const docIterator = await PackManager.promises.makeDocIterator(docId) - const getUpdate = util.promisify(docIterator.next).bind(docIterator) - - const lastUpdate = await getUpdate() - if (!lastUpdate) { - return null - } - - const lastUpdateVersion = lastUpdate.v - - let latestContent - let version - try { - ;[latestContent, version] = await getLatestContent( - projectId, - docId, - lastUpdateVersion - ) - } catch (err) { - if (err instanceof Errors.NotFoundError) { - // Doc not found in docstore. We can't build its history - return null - } else { - throw err - } - } - log.start('load-doc', latestContent, version) - - let content = latestContent - let update = lastUpdate - let previousUpdate = null - - while (update) { - if (packsAreDuplicated(update, previousUpdate)) { - previousUpdate = update - update = await getUpdate() - continue - } - log.update('rewinding', update) - for (let i = update.op.length - 1; i >= 0; i--) { - const op = update.op[i] - if (op.broken === true) { - log.op('skipped', content, op) - continue - } - try { - log.op('rewindOp', content, op) - content = rewindOp(content, op, log) - } catch (e) { - if (e instanceof ConsistencyError && (i = update.op.length - 1)) { - // catch known case where the last op in an array has been - // merged into a later op - op.broken = true - log.opError('marking broken', content, op) - } else { - log.opError('failed', content, op, e) - } - } - } - previousUpdate = update - update = await getUpdate() - } - return log.end() -} - -async function main() { - // Get a list of projects to migrate - let projectIds = [] - if (argv.all) { - const projectsToMigrate = await findProjects( - { 'overleaf.history.display': { $ne: true } }, - { _id: 1, overleaf: 1 } - ) - projectIds = projectsToMigrate.map(p => p._id.toString()) - console.log('Unmigrated projects', projectIds.length) - } else if (argv['project-id']) { - projectIds = [argv['project-id']] - } else { - usage() - process.exit(1) - } - let errorCount = 0 - for (const projectId of projectIds) { - const docIds = await PackManager.promises.findAllDocsInProject(projectId) - if (!docIds.length) { - console.log('No docs found for project', projectId) - } - let projectErrorCount = 0 - for (const docId of docIds) { - const result = await rewindDoc(projectId, docId) - const failed = result.filter(r => r.status === 'failed') - errorCount += failed.length - if (argv.verbose) { - console.log(JSON.stringify({ projectId, docId, result }, null, 2)) - } else if (failed.length > 0) { - console.log('project', projectId, 'docId', docId, 'FAILED') - for (const f of failed) { - console.log(JSON.stringify(f)) - } - projectErrorCount += failed.length - } - } - if (projectErrorCount === 0 && !argv.verbose) { - console.log('project', projectId, 'docs', docIds.length, 'OK') - } - } - process.exit(errorCount > 0 ? 1 : 0) -} - -waitForDb() - .then(TrackChangesMongoDb.waitForDb) - .then(main) - .catch(err => { - console.error(err) - process.exit(1) - }) diff --git a/services/web/scripts/history/downgrade_project.js b/services/web/scripts/history/downgrade_project.js deleted file mode 100644 index b5eec3f297..0000000000 --- a/services/web/scripts/history/downgrade_project.js +++ /dev/null @@ -1,81 +0,0 @@ -const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true' -const DRY_RUN = process.env.DRY_RUN !== 'false' -process.env.MONGO_SOCKET_TIMEOUT = - parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 - -const PROJECT_ID = process.env.PROJECT_ID - -const { ObjectId } = require('mongodb') -const { - db, - waitForDb, - READ_PREFERENCE_SECONDARY, -} = require('../../app/src/infrastructure/mongodb') -const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler') - -console.log({ - DRY_RUN, - VERBOSE_LOGGING, - PROJECT_ID, -}) - -let INTERRUPT = false - -async function processProject(project) { - if (INTERRUPT) { - return - } - if (!shouldPreserveHistory(project)) { - console.log( - `project ${project._id} does not have preserveHistory:true, skipping` - ) - return - } - if (!DRY_RUN) { - await ProjectHistoryHandler.promises.downgradeHistory(project._id) - } - if (VERBOSE_LOGGING) { - console.log(`project ${project._id} downgraded to track-changes`) - } -} - -async function shouldPreserveHistory(project) { - return await db.projectHistoryMetaData.findOne( - { - $and: [ - { project_id: { $eq: project._id } }, - { preserveHistory: { $eq: true } }, - ], - }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) -} - -async function main() { - if (PROJECT_ID) { - await waitForDb() - const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) }) - await processProject(project) - } else { - console.log('PROJECT_ID environment value is needed.') - process.exit(1) - } -} - -// Upgrading history is not atomic, if we quit out mid-initialisation -// then history could get into a broken state -// Instead, skip any unprocessed projects and exit() at end of the batch. -process.on('SIGINT', function () { - console.log('Caught SIGINT, waiting for in process downgrades to complete') - INTERRUPT = true -}) - -main() - .then(() => { - console.error('Done.') - process.exit(0) - }) - .catch(error => { - console.error({ error }) - process.exit(1) - }) diff --git a/services/web/scripts/history/migrate_history.js b/services/web/scripts/history/migrate_history.js deleted file mode 100644 index 589240718b..0000000000 --- a/services/web/scripts/history/migrate_history.js +++ /dev/null @@ -1,287 +0,0 @@ -// raise mongo timeout to 1hr if otherwise unspecified -process.env.MONGO_SOCKET_TIMEOUT = - parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 - -const fs = require('fs') - -if (fs.existsSync('/etc/container_environment.json')) { - try { - const envData = JSON.parse( - fs.readFileSync('/etc/container_environment.json', 'utf8') - ) - for (const [key, value] of Object.entries(envData)) { - process.env[key] = value - } - } catch (err) { - console.error( - 'cannot read /etc/container_environment.json, the script needs to be run as root', - err - ) - process.exit(1) - } -} - -const VERSION = '0.9.0-cli' -const { - countProjects, - countDocHistory, - upgradeProject, - findProjects, -} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper') -const { waitForDb } = require('../../app/src/infrastructure/mongodb') -const minimist = require('minimist') -const util = require('util') -const pLimit = require('p-limit') -const logger = require('@overleaf/logger') -logger.initialize('history-migration') -// disable logging to stdout from internal modules -logger.logger.streams = [] - -const DEFAULT_OUTPUT_FILE = `history-migration-${new Date() - .toISOString() - .replace(/[:.]/g, '_')}.log` - -const argv = minimist(process.argv.slice(2), { - boolean: [ - 'verbose', - 'fix-invalid-characters', - 'convert-large-docs-to-file', - 'import-broken-history-as-zip', - 'force-upgrade-on-failure', - 'dry-run', - 'use-query-hint', - 'retry-failed', - 'archive-on-failure', - 'force-clean', - ], - string: ['output', 'user-id'], - alias: { - verbose: 'v', - output: 'o', - 'dry-run': 'd', - concurrency: 'j', - 'use-query-hint': 'q', - 'retry-failed': 'r', - 'archive-on-failure': 'a', - }, - default: { - output: DEFAULT_OUTPUT_FILE, - concurrency: 1, - 'batch-size': 100, - 'max-upgrades-to-attempt': false, - 'max-failures': 50, - }, -}) - -let INTERRUPT = false - -async function findProjectsToMigrate() { - console.log('History Migration Statistics') - - // Show statistics about the number of projects to migrate - const migratedProjects = await countProjects({ - 'overleaf.history.display': true, - }) - const totalProjects = await countProjects() - console.log('Migrated Projects : ', migratedProjects) - console.log('Total Projects : ', totalProjects) - console.log('Remaining Projects : ', totalProjects - migratedProjects) - - if (migratedProjects === totalProjects) { - console.log('All projects have been migrated') - process.exit(0) - } - - // Get a list of projects to migrate - const projectsToMigrate = await findProjects( - { 'overleaf.history.display': { $ne: true } }, - { _id: 1, overleaf: 1 } - ) - - // Show statistics for docHistory collection - const docHistoryWithoutProjectId = await countDocHistory({ - project_id: { $exists: false }, - }) - - if (docHistoryWithoutProjectId > 0) { - console.log( - `WARNING: docHistory collection contains ${docHistoryWithoutProjectId} records without project_id` - ) - process.exit(1) - } - - return projectsToMigrate -} - -function createProgressBar() { - const startTime = new Date() - return function progressBar(current, total, msg) { - const barLength = 20 - const percentage = Math.floor((current / total) * 100) - const bar = '='.repeat(percentage / (100 / barLength)) - const empty = ' '.repeat(barLength - bar.length) - const elapsed = new Date() - startTime - // convert elapsed time to hours, minutes, seconds - const ss = Math.floor((elapsed / 1000) % 60) - .toString() - .padStart(2, '0') - const mm = Math.floor((elapsed / (1000 * 60)) % 60) - .toString() - .padStart(2, '0') - const hh = Math.floor(elapsed / (1000 * 60 * 60)) - .toString() - .padStart(2, '0') - process.stdout.write( - `\r${hh}:${mm}:${ss} |${bar}${empty}| ${percentage}% (${current}/${total}) ${msg}` - ) - } -} - -async function migrateProjects(projectsToMigrate) { - let projectsMigrated = 0 - let projectsFailed = 0 - - console.log('Starting migration...') - if (argv.concurrency > 1) { - console.log(`Using ${argv.concurrency} concurrent migrations`) - } - // send log output for each migration to a file - const output = fs.createWriteStream(argv.output, { flags: 'a' }) - console.log(`Writing log output to ${process.cwd()}/${argv.output}`) - const logger = new console.Console({ stdout: output }) - function logJson(obj) { - logger.log(JSON.stringify(obj)) - } - // limit the number of concurrent migrations - const limit = pLimit(argv.concurrency) - const jobs = [] - // throttle progress reporting to 2x per second - const progressBar = createProgressBar() - let i = 0 - const N = projectsToMigrate.length - const progressBarTimer = setInterval(() => { - if (INTERRUPT) { - return // don't update the progress bar if we're shutting down - } - progressBar( - i, - N, - `Migrated: ${projectsMigrated}, Failed: ${projectsFailed}` - ) - }, 500) - - const options = { - migrationOptions: { - archiveOnFailure: argv['import-broken-history-as-zip'], - fixInvalidCharacters: argv['fix-invalid-characters'], - forceNewHistoryOnFailure: argv['force-upgrade-on-failure'], - }, - convertLargeDocsToFile: argv['convert-large-docs-to-file'], - userId: argv['user-id'], - reason: VERSION, - forceClean: argv['force-clean'], - } - async function _migrateProject(project) { - if (INTERRUPT) { - return // don't start any new jobs if we're shutting down - } - const startTime = new Date() - try { - const result = await upgradeProject(project._id, options) - i++ - if (INTERRUPT && limit.activeCount > 1) { - // an interrupt was requested while this job was running - // report that we're waiting for the remaining jobs to finish - console.log( - `Waiting for remaining ${ - limit.activeCount - 1 - } active jobs to finish\r` - ) - } - if (result.error) { - // failed to migrate this project - logJson({ - project_id: project._id, - result, - stack: result.error.stack, - startTime, - endTime: new Date(), - }) - projectsFailed++ - } else { - // successfully migrated this project - logJson({ - project_id: project._id, - result, - startTime, - endTime: new Date(), - }) - projectsMigrated++ - } - } catch (err) { - // unexpected error from the migration - projectsFailed++ - logJson({ - project_id: project._id, - exception: util.inspect(err), - startTime, - endTime: new Date(), - }) - } - } - - for (const project of projectsToMigrate) { - jobs.push(limit(_migrateProject, project)) - } - // wait for all the queued jobs to complete - await Promise.all(jobs) - clearInterval(progressBarTimer) - progressBar(i, N, `Migrated: ${projectsMigrated}, Failed: ${projectsFailed}`) - process.stdout.write('\n') - return { projectsMigrated, projectsFailed } -} - -async function main() { - const projectsToMigrate = await findProjectsToMigrate() - if (argv['dry-run']) { - console.log('Dry run, exiting') - process.exit(0) - } - const { projectsMigrated, projectsFailed } = await migrateProjects( - projectsToMigrate - ) - console.log('Projects migrated: ', projectsMigrated) - console.log('Projects failed: ', projectsFailed) - if (projectsFailed > 0) { - console.log('------------------------------------------------------') - console.log(`Log output written to ${process.cwd()}/${argv.output}`) - console.log( - 'Please check the log for errors. Attach the content of the file when contacting support.' - ) - console.log('------------------------------------------------------') - } - if (INTERRUPT) { - console.log('Migration interrupted, please run again to continue.') - } else if (projectsFailed === 0) { - console.log(`All projects migrated successfully.`) - } - console.log('Done.') - process.exit(projectsFailed > 0 ? 1 : 0) -} - -// Upgrading history is not atomic, if we quit out mid-initialisation -// then history could get into a broken state -// Instead, skip any unprocessed projects and exit() at end of the batch. -process.on('SIGINT', function () { - console.log( - '\nCaught SIGINT, waiting for all in-progess upgrades to complete' - ) - INTERRUPT = true -}) - -waitForDb() - .then(main) - .catch(err => { - console.error(err) - process.exit(1) - }) diff --git a/services/web/scripts/history/reset_incorrect_doc_revision.js b/services/web/scripts/history/reset_incorrect_doc_revision.js deleted file mode 100644 index 805a7ad8f2..0000000000 --- a/services/web/scripts/history/reset_incorrect_doc_revision.js +++ /dev/null @@ -1,111 +0,0 @@ -const DRY_RUN = process.env.DRY_RUN !== 'false' -const PROJECT_ID = process.env.PROJECT_ID -const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true' -const VERBOSE_PROJECT_NAMES = process.env.VERBOSE_PROJECT_NAMES === 'true' -const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 50 -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 500 -// persist fallback in order to keep batchedUpdate in-sync -process.env.BATCH_SIZE = BATCH_SIZE -process.env.VERBOSE_LOGGING = VERBOSE_LOGGING - -const { ObjectId } = require('mongodb') -const { db, waitForDb } = require('../../app/src/infrastructure/mongodb') -const { batchedUpdate } = require('../helpers/batchedUpdate') -const { promiseMapWithLimit } = require('../../app/src/util/promises') - -const count = { - projects: 0, - projectsWithIncorrectRevDocs: 0, - totalIncorrectRevDocs: 0, - totalNanRevDocs: 0, - totalNullRevDocs: 0, - totalUndefinedRevDocs: 0, - convertedRevs: 0, -} - -async function main() { - const projection = { - _id: 1, - } - - if (VERBOSE_PROJECT_NAMES) { - projection.name = 1 - } - - const options = {} - - if (PROJECT_ID) { - const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) }) - await processProject(project) - } else { - await batchedUpdate( - 'projects', - { 'overleaf.history.display': { $ne: true } }, - processBatch, - projection, - options - ) - } - console.log('Final') -} - -async function processBatch(projects) { - await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject) -} - -async function processProject(project) { - count.projects++ - - const docs = await db.docs - .find( - { - project_id: project._id, - $or: [{ rev: null }, { rev: NaN }], - }, - { _id: 1, rev: 1 } - ) - .toArray() - - if (!docs || docs.length <= 0) { - return - } - - if (VERBOSE_LOGGING) { - console.log( - `Found ${docs.length} incorrect doc.rev for project ${ - project[VERBOSE_PROJECT_NAMES ? 'name' : '_id'] - }` - ) - } - - count.projectsWithIncorrectRevDocs++ - count.totalIncorrectRevDocs += docs.length - - for (const doc of docs) { - if (doc.rev === undefined) { - count.totalUndefinedRevDocs++ - } else if (doc.rev === null) { - count.totalNullRevDocs++ - } else if (isNaN(doc.rev)) { - count.totalNanRevDocs++ - } else { - console.error(`unknown 'rev' value: ${doc.rev}`) - } - if (!DRY_RUN) { - console.log(`fixing rev of doc ${doc.id} from '${doc.rev}' to 0`) - await db.docs.updateOne({ _id: doc._id }, { $set: { rev: 0 } }) - count.convertedRevs++ - } - } -} - -waitForDb() - .then(main) - .then(() => { - console.log(count) - process.exit(0) - }) - .catch(err => { - console.log('Something went wrong!', err) - process.exit(1) - }) diff --git a/services/web/scripts/history/unset_allow_downgrade.js b/services/web/scripts/history/unset_allow_downgrade.js deleted file mode 100644 index d6dc5a03a4..0000000000 --- a/services/web/scripts/history/unset_allow_downgrade.js +++ /dev/null @@ -1,133 +0,0 @@ -const { promisify } = require('util') -const { ObjectId } = require('mongodb') -const { - db, - waitForDb, - READ_PREFERENCE_SECONDARY, -} = require('../../app/src/infrastructure/mongodb') -const sleep = promisify(setTimeout) -const _ = require('lodash') - -const NOW_IN_S = Date.now() / 1000 -const ONE_WEEK_IN_S = 60 * 60 * 24 * 7 -const TEN_SECONDS = 10 * 1000 - -function getSecondsFromObjectId(id) { - return id.getTimestamp().getTime() / 1000 -} - -async function main(options) { - if (!options) { - options = {} - } - _.defaults(options, { - projectId: process.env.PROJECT_ID, - dryRun: process.env.DRY_RUN !== 'false', - verboseLogging: process.env.VERBOSE_LOGGING === 'true', - firstProjectId: process.env.FIRST_PROJECT_ID - ? ObjectId(process.env.FIRST_PROJECT_ID) - : ObjectId('4b3d3b3d0000000000000000'), // timestamped to 2010-01-01T00:01:01.000Z - incrementByS: parseInt(process.env.INCREMENT_BY_S, 10) || ONE_WEEK_IN_S, - batchSize: parseInt(process.env.BATCH_SIZE, 10) || 1000, - stopAtS: parseInt(process.env.STOP_AT_S, 10) || NOW_IN_S, - letUserDoubleCheckInputsFor: - parseInt(process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR, 10) || TEN_SECONDS, - }) - - if (options.projectId) { - await waitForDb() - const { modifiedCount } = await db.projects.updateOne( - { - _id: ObjectId(options.projectId), - 'overleaf.history.allowDowngrade': true, - }, - { $unset: { 'overleaf.history.allowDowngrade': 1 } } - ) - console.log(`modifiedCount: ${modifiedCount}`) - process.exit(0) - } - - await letUserDoubleCheckInputs(options) - await waitForDb() - - let startId = options.firstProjectId - - let totalProcessed = 0 - while (getSecondsFromObjectId(startId) <= options.stopAtS) { - let batchProcessed = 0 - const end = getSecondsFromObjectId(startId) + options.incrementByS - let endId = ObjectId.createFromTime(end) - const query = { - _id: { - // include edge - $gte: startId, - // exclude edge - $lt: endId, - }, - 'overleaf.history.allowDowngrade': true, - } - const projects = await db.projects - .find(query, { readPreference: READ_PREFERENCE_SECONDARY }) - .project({ _id: 1 }) - .limit(options.batchSize) - .toArray() - - if (projects.length) { - const projectIds = projects.map(project => project._id) - if (options.verboseLogging) { - console.log( - `Processing projects with ids: ${JSON.stringify(projectIds)}` - ) - } else { - console.log(`Processing ${projects.length} projects`) - } - - if (!options.dryRun) { - await db.projects.updateMany( - { _id: { $in: projectIds } }, - { $unset: { 'overleaf.history.allowDowngrade': 1 } } - ) - } else { - console.log( - `skipping update of ${projectIds.length} projects in dry-run mode` - ) - } - - totalProcessed += projectIds.length - batchProcessed += projectIds.length - - if (projects.length === options.batchSize) { - endId = projects[projects.length - 1]._id - } - } - console.error( - `Processed ${batchProcessed} from ${startId} until ${endId} (${totalProcessed} processed in total)` - ) - - startId = endId - } -} - -async function letUserDoubleCheckInputs(options) { - console.error('Options:', JSON.stringify(options, null, 2)) - console.error( - 'Waiting for you to double check inputs for', - options.letUserDoubleCheckInputsFor, - 'ms' - ) - await sleep(options.letUserDoubleCheckInputsFor) -} - -module.exports = main - -if (require.main === module) { - main() - .then(() => { - console.error('Done.') - process.exit(0) - }) - .catch(error => { - console.error({ error }) - process.exit(1) - }) -} diff --git a/services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js b/services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js deleted file mode 100644 index b0b91cca50..0000000000 --- a/services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js +++ /dev/null @@ -1,208 +0,0 @@ -const SCRIPT_VERSION = 4 -const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true' -const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10 -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100 -const DRY_RUN = process.env.DRY_RUN !== 'false' -const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false' -const RETRY_FAILED = process.env.RETRY_FAILED === 'true' -const MAX_UPGRADES_TO_ATTEMPT = - parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false -const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50 -const ARCHIVE_ON_FAILURE = process.env.ARCHIVE_ON_FAILURE === 'true' -const FIX_INVALID_CHARACTERS = process.env.FIX_INVALID_CHARACTERS === 'true' -const FORCE_NEW_HISTORY_ON_FAILURE = - process.env.FORCE_NEW_HISTORY_ON_FAILURE === 'true' -const IMPORT_ZIP_FILE_PATH = process.env.IMPORT_ZIP_FILE_PATH -const CUTOFF_DATE = process.env.CUTOFF_DATE - ? new Date(process.env.CUTOFF_DATE) - : undefined -// persist fallback in order to keep batchedUpdate in-sync -process.env.BATCH_SIZE = BATCH_SIZE -// raise mongo timeout to 1hr if otherwise unspecified -process.env.MONGO_SOCKET_TIMEOUT = - parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 - -const PROJECT_ID = process.env.PROJECT_ID - -// User id is required to move large documents to filestore -const USER_ID = process.env.USER_ID -const CONVERT_LARGE_DOCS_TO_FILE = - process.env.CONVERT_LARGE_DOCS_TO_FILE === 'true' - -const { ObjectId } = require('mongodb') -const { db, waitForDb } = require('../../app/src/infrastructure/mongodb') -const { promiseMapWithLimit } = require('../../app/src/util/promises') -const { batchedUpdate } = require('../helpers/batchedUpdate') -const { - anyDocHistoryExists, - anyDocHistoryIndexExists, - doUpgradeForNoneWithConversion, -} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper') - -console.log({ - DRY_RUN, - VERBOSE_LOGGING, - WRITE_CONCURRENCY, - BATCH_SIZE, - MAX_UPGRADES_TO_ATTEMPT, - MAX_FAILURES, - USE_QUERY_HINT, - RETRY_FAILED, - ARCHIVE_ON_FAILURE, - PROJECT_ID, - FIX_INVALID_CHARACTERS, - FORCE_NEW_HISTORY_ON_FAILURE, - CONVERT_LARGE_DOCS_TO_FILE, - USER_ID, - IMPORT_ZIP_FILE_PATH, - CUTOFF_DATE, -}) - -const RESULT = { - DRY_RUN, - attempted: 0, - projectsUpgraded: 0, - failed: 0, - continueFrom: null, -} - -let INTERRUPT = false - -async function processBatch(projects) { - if (projects.length && projects[0]._id) { - RESULT.continueFrom = projects[0]._id - } - await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject) - console.log(RESULT) - if (INTERRUPT) { - // ctrl+c - console.log('Terminated by SIGINT') - process.exit(0) - } - if (RESULT.failed >= MAX_FAILURES) { - console.log(`MAX_FAILURES limit (${MAX_FAILURES}) reached. Stopping.`) - process.exit(0) - } - if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) { - console.log( - `MAX_UPGRADES_TO_ATTEMPT limit (${MAX_UPGRADES_TO_ATTEMPT}) reached. Stopping.` - ) - process.exit(0) - } -} - -async function processProject(project) { - if (INTERRUPT) { - return - } - if (project.overleaf && project.overleaf.history) { - // projects we're upgrading like this should never have a history id - if (project.overleaf.history.id) { - return - } - if ( - project.overleaf.history.conversionFailed || - project.overleaf.history.upgradeFailed - ) { - if (project.overleaf.history.zipFileArchivedInProject) { - return // always give up if we have uploaded the zipfile to the project - } - if (!RETRY_FAILED) { - // we don't want to attempt upgrade on projects - // that have been previously attempted and failed - return - } - } - } - if (RESULT.failed >= MAX_FAILURES) { - return - } - if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) { - return - } - const anyDocHistoryOrIndex = - (await anyDocHistoryExists(project)) || - (await anyDocHistoryIndexExists(project)) - if (anyDocHistoryOrIndex) { - RESULT.attempted += 1 - if (DRY_RUN) { - return - } - const result = await doUpgradeForNoneWithConversion(project, { - migrationOptions: { - archiveOnFailure: ARCHIVE_ON_FAILURE, - fixInvalidCharacters: FIX_INVALID_CHARACTERS, - forceNewHistoryOnFailure: FORCE_NEW_HISTORY_ON_FAILURE, - importZipFilePath: IMPORT_ZIP_FILE_PATH, - cutoffDate: CUTOFF_DATE, - }, - convertLargeDocsToFile: CONVERT_LARGE_DOCS_TO_FILE, - userId: USER_ID, - reason: `${SCRIPT_VERSION}`, - }) - if (result.convertedDocCount) { - console.log( - `project ${project._id} converted ${result.convertedDocCount} docs to filestore` - ) - } - if (result.error) { - console.error(`project ${project._id} FAILED with error: `, result.error) - RESULT.failed += 1 - } else if (result.upgraded) { - if (VERBOSE_LOGGING) { - console.log( - `project ${project._id} converted and upgraded to full project history` - ) - } - RESULT.projectsUpgraded += 1 - } - } -} - -async function main() { - if (PROJECT_ID) { - await waitForDb() - const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) }) - await processProject(project) - } else { - const projection = { - _id: 1, - overleaf: 1, - } - const options = {} - if (USE_QUERY_HINT) { - options.hint = { _id: 1 } - } - await batchedUpdate( - 'projects', - // we originally used - // 'overleaf.history.id': { $exists: false } - // but display false is indexed and contains all the above, - // it can be faster to skip projects with a history ID than to use a query - { 'overleaf.history.display': { $ne: true } }, - processBatch, - projection, - options - ) - } - console.log('Final') - console.log(RESULT) -} - -// Upgrading history is not atomic, if we quit out mid-initialisation -// then history could get into a broken state -// Instead, skip any unprocessed projects and exit() at end of the batch. -process.on('SIGINT', function () { - console.log('Caught SIGINT, waiting for in process upgrades to complete') - INTERRUPT = true -}) - -main() - .then(() => { - console.error('Done.') - process.exit(0) - }) - .catch(error => { - console.error({ error }) - process.exit(1) - }) diff --git a/services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js b/services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js deleted file mode 100644 index 7a74527348..0000000000 --- a/services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js +++ /dev/null @@ -1,232 +0,0 @@ -const SCRIPT_VERSION = 3 -const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true' -const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10 -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100 -const DRY_RUN = process.env.DRY_RUN !== 'false' -const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false' -const RETRY_FAILED = process.env.RETRY_FAILED === 'true' -const MAX_UPGRADES_TO_ATTEMPT = - parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false -const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50 -// persist fallback in order to keep batchedUpdate in-sync -process.env.BATCH_SIZE = BATCH_SIZE -// raise mongo timeout to 1hr if otherwise unspecified -process.env.MONGO_SOCKET_TIMEOUT = - parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 - -const { - db, - READ_PREFERENCE_SECONDARY, -} = require('../../app/src/infrastructure/mongodb') -const { promiseMapWithLimit } = require('../../app/src/util/promises') -const { batchedUpdate } = require('../helpers/batchedUpdate') -const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler') -const HistoryManager = require('../../app/src/Features/History/HistoryManager') - -console.log({ - DRY_RUN, - VERBOSE_LOGGING, - WRITE_CONCURRENCY, - BATCH_SIZE, - MAX_UPGRADES_TO_ATTEMPT, - MAX_FAILURES, - USE_QUERY_HINT, - RETRY_FAILED, -}) - -const RESULT = { - DRY_RUN, - attempted: 0, - projectsUpgraded: 0, - failed: 0, - continueFrom: null, -} - -let INTERRUPT = false - -async function processBatch(projects) { - if (projects.length && projects[0]._id) { - RESULT.continueFrom = projects[0]._id - } - await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject) - console.log(RESULT) - if (INTERRUPT) { - // ctrl+c - console.log('Terminated by SIGINT') - process.exit(0) - } - if (RESULT.failed >= MAX_FAILURES) { - console.log(`MAX_FAILURES limit (${MAX_FAILURES}) reached. Stopping.`) - process.exit(0) - } - if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) { - console.log( - `MAX_UPGRADES_TO_ATTEMPT limit (${MAX_UPGRADES_TO_ATTEMPT}) reached. Stopping.` - ) - process.exit(0) - } -} - -async function processProject(project) { - if (INTERRUPT) { - return - } - // If upgradeFailed, skip unless we're explicitly retrying failed upgrades - if ( - project.overleaf && - project.overleaf.history && - project.overleaf.history.upgradeFailed - ) { - if (RETRY_FAILED) { - return await doUpgradeForNoneWithoutConversion(project) - } else { - return - } - } - // Skip any projects with a history ID, these are v1 - if ( - project.overleaf && - project.overleaf.history && - project.overleaf.history.id - ) { - return - } - const anyDocHistory = await anyDocHistoryExists(project) - if (anyDocHistory) { - return - } - const anyDocHistoryIndex = await anyDocHistoryIndexExists(project) - if (anyDocHistoryIndex) { - return - } - await doUpgradeForNoneWithoutConversion(project) -} - -async function doUpgradeForNoneWithoutConversion(project) { - if (RESULT.failed >= MAX_FAILURES) { - return - } - if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) { - return - } - RESULT.attempted += 1 - const projectId = project._id - if (!DRY_RUN) { - // ensureHistoryExistsForProject resyncs project - // Change to 'peek'ing the doc when resyncing should - // be rolled out prior to using this script - try { - // Logic originally from ProjectHistoryHandler.ensureHistoryExistsForProject - // However sends a force resync project to project history instead - // of a resync request to doc-updater - let historyId = await ProjectHistoryHandler.promises.getHistoryId( - projectId - ) - if (historyId == null) { - historyId = await HistoryManager.promises.initializeProject(projectId) - if (historyId != null) { - await ProjectHistoryHandler.promises.setHistoryId( - projectId, - historyId - ) - } - } - await HistoryManager.promises.resyncProject(projectId, { - force: true, - origin: { kind: 'history-migration' }, - }) - await HistoryManager.promises.flushProject(projectId) - } catch (err) { - RESULT.failed += 1 - console.error(`project ${project._id} FAILED with error: `, err) - await db.projects.updateOne( - { _id: project._id }, - { - $set: { - 'overleaf.history.upgradeFailed': true, - }, - } - ) - return - } - await db.projects.updateOne( - { _id: project._id }, - { - $set: { - 'overleaf.history.display': true, - 'overleaf.history.upgradedAt': new Date(), - 'overleaf.history.upgradeReason': `none-without-sl-history/${SCRIPT_VERSION}`, - }, - $unset: { - 'overleaf.history.upgradeFailed': true, - }, - } - ) - } - if (VERBOSE_LOGGING) { - console.log(`project ${project._id} converted to full project history`) - } - RESULT.projectsUpgraded += 1 -} - -async function anyDocHistoryExists(project) { - return await db.docHistory.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: READ_PREFERENCE_SECONDARY, - } - ) -} - -async function anyDocHistoryIndexExists(project) { - return await db.docHistoryIndex.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: READ_PREFERENCE_SECONDARY, - } - ) -} - -async function main() { - const projection = { - _id: 1, - overleaf: 1, - } - const options = {} - if (USE_QUERY_HINT) { - options.hint = { _id: 1 } - } - await batchedUpdate( - 'projects', - // we originally used - // 'overleaf.history.id': { $exists: false } - // but display false is indexed and contains all the above, - // plus we want to be able to retry failed upgrades with a history id - { 'overleaf.history.display': { $ne: true } }, - processBatch, - projection, - options - ) - console.log('Final') - console.log(RESULT) -} - -// Upgrading history is not atomic, if we quit out mid-initialisation -// then history could get into a broken state -// Instead, skip any unprocessed projects and exit() at end of the batch. -process.on('SIGINT', function () { - console.log('Caught SIGINT, waiting for in process upgrades to complete') - INTERRUPT = true -}) - -main() - .then(() => { - console.error('Done.') - process.exit(0) - }) - .catch(error => { - console.error({ error }) - process.exit(1) - }) diff --git a/services/web/scripts/history/upgrade_project.js b/services/web/scripts/history/upgrade_project.js deleted file mode 100644 index b36cdb76e4..0000000000 --- a/services/web/scripts/history/upgrade_project.js +++ /dev/null @@ -1,44 +0,0 @@ -const { ObjectId } = require('mongodb') -const { - db, - waitForDb, - READ_PREFERENCE_SECONDARY, -} = require('../../app/src/infrastructure/mongodb') -const { - upgradeProject, -} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper') - -async function processProject(project) { - const result = await upgradeProject(project) - console.log(result) -} - -async function main() { - await waitForDb() - const args = process.argv.slice(2) - const projectId = args[0] - const query = { _id: ObjectId(projectId) } - const projection = { - _id: 1, - overleaf: 1, - } - const options = { - projection, - readPreference: READ_PREFERENCE_SECONDARY, - } - const project = await db.projects.findOne(query, options) - if (project) { - await processProject(project) - } else { - console.error(`project ${projectId} not found`) - } -} - -main() - .then(() => { - process.exit(0) - }) - .catch(error => { - console.error({ error }) - process.exit(1) - }) diff --git a/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js b/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js deleted file mode 100644 index 7ae69bad24..0000000000 --- a/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js +++ /dev/null @@ -1,243 +0,0 @@ -const SCRIPT_VERSION = 1 -const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true' -const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10 -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100 -const DRY_RUN = process.env.DRY_RUN !== 'false' -const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false' -const RETRY_FAILED = process.env.RETRY_FAILED === 'true' -const MAX_UPGRADES_TO_ATTEMPT = - parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false -const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50 -// persist fallback in order to keep batchedUpdate in-sync -process.env.BATCH_SIZE = BATCH_SIZE -// raise mongo timeout to 1hr if otherwise unspecified -process.env.MONGO_SOCKET_TIMEOUT = - parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 - -const PROJECT_ID = process.env.PROJECT_ID - -const { ObjectId } = require('mongodb') -const { - db, - waitForDb, - READ_PREFERENCE_SECONDARY, -} = require('../../app/src/infrastructure/mongodb') -const { promiseMapWithLimit } = require('../../app/src/util/promises') -const { batchedUpdate } = require('../helpers/batchedUpdate') -const ProjectHistoryController = require('../../modules/history-migration/app/src/ProjectHistoryController') - -console.log({ - DRY_RUN, - VERBOSE_LOGGING, - WRITE_CONCURRENCY, - BATCH_SIZE, - MAX_UPGRADES_TO_ATTEMPT, - MAX_FAILURES, - USE_QUERY_HINT, - RETRY_FAILED, - PROJECT_ID, -}) - -const RESULT = { - DRY_RUN, - attempted: 0, - projectsUpgraded: 0, - failed: 0, - continueFrom: null, -} - -let INTERRUPT = false - -async function processBatch(projects) { - if (projects.length && projects[0]._id) { - RESULT.continueFrom = projects[0]._id - } - await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject) - console.log(RESULT) - if (INTERRUPT) { - // ctrl+c - console.log('Terminated by SIGINT') - process.exit(0) - } - if (RESULT.failed >= MAX_FAILURES) { - console.log(`MAX_FAILURES limit (${MAX_FAILURES}) reached. Stopping.`) - process.exit(0) - } - if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) { - console.log( - `MAX_UPGRADES_TO_ATTEMPT limit (${MAX_UPGRADES_TO_ATTEMPT}) reached. Stopping.` - ) - process.exit(0) - } -} - -async function processProject(project) { - if (INTERRUPT) { - return - } - // skip safety check if we want to retry failed upgrades - if (!RETRY_FAILED) { - if (project.overleaf && project.overleaf.history) { - if ( - project.overleaf.history.conversionFailed || - project.overleaf.history.upgradeFailed - ) { - // we don't want to attempt upgrade on projects - // that have been previously attempted and failed - return - } - } - } - const preserveHistory = await shouldPreserveHistory(project) - if (preserveHistory) { - const anyDocHistory = await anyDocHistoryExists(project) - if (anyDocHistory) { - return await doUpgradeForV1WithConversion(project) - } - const anyDocHistoryIndex = await anyDocHistoryIndexExists(project) - if (anyDocHistoryIndex) { - return await doUpgradeForV1WithConversion(project) - } - } -} - -async function doUpgradeForV1WithConversion(project) { - if (RESULT.failed >= MAX_FAILURES) { - return - } - if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) { - return - } - RESULT.attempted += 1 - const projectId = project._id - // migrateProjectHistory expects project id as a string - const projectIdString = project._id.toString() - if (!DRY_RUN) { - try { - // We treat these essentially as None projects, the V1 history is irrelevant, - // so we will delete it, and do a conversion as if we're a None project - await ProjectHistoryController.deleteProjectHistory(projectIdString) - if (VERBOSE_LOGGING) { - console.log( - `project ${projectId} existing full project history deleted` - ) - } - await ProjectHistoryController.migrateProjectHistory(projectIdString) - } catch (err) { - // if migrateProjectHistory fails, it cleans up by deleting - // the history and unsetting the history id - // therefore a failed project will still look like a 'None with conversion' project - RESULT.failed += 1 - console.error(`project ${projectId} FAILED with error: `, err) - // We set a failed flag so future runs of the script don't automatically retry - await db.projects.updateOne( - { _id: projectId }, - { - $set: { - 'overleaf.history.conversionFailed': true, - }, - } - ) - return - } - await db.projects.updateOne( - { _id: projectId }, - { - $set: { - 'overleaf.history.upgradeReason': `v1-with-conversion/${SCRIPT_VERSION}`, - }, - $unset: { - 'overleaf.history.upgradeFailed': true, - 'overleaf.history.conversionFailed': true, - }, - } - ) - } - if (VERBOSE_LOGGING) { - console.log( - `project ${projectId} converted and upgraded to full project history` - ) - } - RESULT.projectsUpgraded += 1 -} - -async function shouldPreserveHistory(project) { - return await db.projectHistoryMetaData.findOne( - { - $and: [ - { project_id: { $eq: project._id } }, - { preserveHistory: { $eq: true } }, - ], - }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) -} - -async function anyDocHistoryExists(project) { - return await db.docHistory.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: READ_PREFERENCE_SECONDARY, - } - ) -} - -async function anyDocHistoryIndexExists(project) { - return await db.docHistoryIndex.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: READ_PREFERENCE_SECONDARY, - } - ) -} - -async function main() { - if (PROJECT_ID) { - await waitForDb() - const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) }) - await processProject(project) - } else { - const projection = { - _id: 1, - overleaf: 1, - } - const options = {} - if (USE_QUERY_HINT) { - options.hint = { _id: 1 } - } - await batchedUpdate( - 'projects', - { - $and: [ - { 'overleaf.history.display': { $ne: true } }, - { 'overleaf.history.id': { $exists: true } }, - ], - }, - processBatch, - projection, - options - ) - console.log('Final') - console.log(RESULT) - } -} - -// Upgrading history is not atomic, if we quit out mid-initialisation -// then history could get into a broken state -// Instead, skip any unprocessed projects and exit() at end of the batch. -process.on('SIGINT', function () { - console.log('Caught SIGINT, waiting for in process upgrades to complete') - INTERRUPT = true -}) - -main() - .then(() => { - console.error('Done.') - process.exit(0) - }) - .catch(error => { - console.error({ error }) - process.exit(1) - }) diff --git a/services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js b/services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js deleted file mode 100644 index f059799969..0000000000 --- a/services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js +++ /dev/null @@ -1,165 +0,0 @@ -const SCRIPT_VERSION = 2 -const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true' -const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10 -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100 -const DRY_RUN = process.env.DRY_RUN !== 'false' -const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false' -// persist fallback in order to keep batchedUpdate in-sync -process.env.BATCH_SIZE = BATCH_SIZE -// raise mongo timeout to 1hr if otherwise unspecified -process.env.MONGO_SOCKET_TIMEOUT = - parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 - -const { ObjectId } = require('mongodb') -const { - db, - READ_PREFERENCE_SECONDARY, -} = require('../../app/src/infrastructure/mongodb') -const { promiseMapWithLimit } = require('../../app/src/util/promises') -const { batchedUpdate } = require('../helpers/batchedUpdate') - -console.log({ - DRY_RUN, - VERBOSE_LOGGING, - WRITE_CONCURRENCY, - BATCH_SIZE, - USE_QUERY_HINT, -}) - -const RESULT = { - DRY_RUN, - projectsUpgraded: 0, -} - -const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = '5a8d8a370000000000000000' -const OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = new ObjectId( - ID_WHEN_FULL_PROJECT_HISTORY_ENABLED -) -const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED = - OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED.getTimestamp() - -// set a default BATCH_LAST_ID at our cutoff point if none set -// we still check against this cut off point later, even if -// BATCH_LAST_ID is set to something problematic -if (!process.env.BATCH_LAST_ID) { - process.env.BATCH_LAST_ID = ID_WHEN_FULL_PROJECT_HISTORY_ENABLED -} - -async function processBatch(projects) { - await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject) - console.log(RESULT) -} - -async function processProject(project) { - // safety check - if ( - project.overleaf && - project.overleaf.history && - project.overleaf.history.upgradeFailed - ) { - // a failed history upgrade might look like a v1 project, but history may be broken - return - } - if (!projectCreatedAfterFullProjectHistoryEnabled(project)) { - return - } - // if they have SL history, continue to send to both history systems (for now) - const anyDocHistory = await anyDocHistoryExists(project) - if (anyDocHistory) { - return await doUpgradeForV1WithoutConversion(project, true) - } - const anyDocHistoryIndex = await anyDocHistoryIndexExists(project) - if (anyDocHistoryIndex) { - return await doUpgradeForV1WithoutConversion(project, true) - } - // or if no sl history, nothing to 'downgrade' to - return await doUpgradeForV1WithoutConversion(project, false) -} - -function projectCreatedAfterFullProjectHistoryEnabled(project) { - return ( - project._id.getTimestamp() >= DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED - ) -} - -async function doUpgradeForV1WithoutConversion(project, allowDowngrade) { - const setProperties = { - 'overleaf.history.display': true, - 'overleaf.history.upgradedAt': new Date(), - 'overleaf.history.upgradeReason': `v1-after-fph/${SCRIPT_VERSION}`, - } - if (allowDowngrade) { - setProperties['overleaf.history.allowDowngrade'] = true - } - if (!DRY_RUN) { - await db.projects.updateOne( - { _id: project._id }, - { - $set: setProperties, - } - ) - } - if (VERBOSE_LOGGING) { - console.log( - `project ${project._id} converted to full project history${ - allowDowngrade ? ', with allowDowngrade' : '' - }` - ) - } - RESULT.projectsUpgraded += 1 -} - -async function anyDocHistoryExists(project) { - return await db.docHistory.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: READ_PREFERENCE_SECONDARY, - } - ) -} - -async function anyDocHistoryIndexExists(project) { - return await db.docHistoryIndex.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: READ_PREFERENCE_SECONDARY, - } - ) -} - -async function main() { - const projection = { - _id: 1, - overleaf: 1, - } - const options = {} - if (USE_QUERY_HINT) { - options.hint = { _id: 1 } - } - await batchedUpdate( - 'projects', - { - $and: [ - { 'overleaf.history.display': { $ne: true } }, - { 'overleaf.history.id': { $exists: true } }, - ], - }, - processBatch, - projection, - options - ) - console.log('Final') - console.log(RESULT) -} - -main() - .then(() => { - console.error('Done.') - process.exit(0) - }) - .catch(error => { - console.error({ error }) - process.exit(1) - }) diff --git a/services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js b/services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js deleted file mode 100644 index 6c88c54464..0000000000 --- a/services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js +++ /dev/null @@ -1,161 +0,0 @@ -const SCRIPT_VERSION = 3 -const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true' -const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10 -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100 -const DRY_RUN = process.env.DRY_RUN !== 'false' -const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false' -const UPGRADE_FAILED_WITH_EMPTY_HISTORY = - process.env.UPGRADE_FAILED_WITH_EMPTY_HISTORY === 'true' -// persist fallback in order to keep batchedUpdate in-sync -process.env.BATCH_SIZE = BATCH_SIZE -// raise mongo timeout to 1hr if otherwise unspecified -process.env.MONGO_SOCKET_TIMEOUT = - parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 - -const { - db, - READ_PREFERENCE_SECONDARY, -} = require('../../app/src/infrastructure/mongodb') -const { promiseMapWithLimit } = require('../../app/src/util/promises') -const { batchedUpdate } = require('../helpers/batchedUpdate') - -console.log({ - DRY_RUN, - VERBOSE_LOGGING, - WRITE_CONCURRENCY, - BATCH_SIZE, - USE_QUERY_HINT, - UPGRADE_FAILED_WITH_EMPTY_HISTORY, -}) - -const RESULT = { - DRY_RUN, - projectsUpgraded: 0, -} - -async function processBatch(projects) { - await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject) - console.log(RESULT) -} - -async function processProject(project) { - // safety check if history exists and there was a failed upgrade - const anyDocHistory = await anyDocHistoryExists(project) - const anyDocHistoryIndex = await anyDocHistoryIndexExists(project) - if ( - project.overleaf && - project.overleaf.history && - project.overleaf.history.upgradeFailed - ) { - const emptyHistory = !anyDocHistory && !anyDocHistoryIndex - if (emptyHistory && UPGRADE_FAILED_WITH_EMPTY_HISTORY) { - console.log( - `upgrading previously failed project ${project._id} with empty history` - ) - } else { - // a failed history upgrade might look like a v1 project, but history may be broken - return - } - } - const preserveHistory = await shouldPreserveHistory(project) - if (preserveHistory) { - // if we need to preserve history, then we must bail out if history exists - if (anyDocHistory) { - return - } - if (anyDocHistoryIndex) { - return - } - return await doUpgradeForV1WithoutConversion(project) - } else { - // if preserveHistory false, then max 7 days of SL history - // but v1 already record to both histories, so safe to upgrade - return await doUpgradeForV1WithoutConversion(project) - } -} - -async function doUpgradeForV1WithoutConversion(project) { - if (!DRY_RUN) { - await db.projects.updateOne( - { _id: project._id }, - { - $set: { - 'overleaf.history.display': true, - 'overleaf.history.upgradedAt': new Date(), - 'overleaf.history.upgradeReason': `v1-without-sl-history/${SCRIPT_VERSION}`, - }, - } - ) - } - if (VERBOSE_LOGGING) { - console.log(`project ${project._id} converted to full project history`) - } - RESULT.projectsUpgraded += 1 -} - -async function shouldPreserveHistory(project) { - return await db.projectHistoryMetaData.findOne( - { - $and: [ - { project_id: { $eq: project._id } }, - { preserveHistory: { $eq: true } }, - ], - }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) -} - -async function anyDocHistoryExists(project) { - return await db.docHistory.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: READ_PREFERENCE_SECONDARY, - } - ) -} - -async function anyDocHistoryIndexExists(project) { - return await db.docHistoryIndex.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: READ_PREFERENCE_SECONDARY, - } - ) -} - -async function main() { - const projection = { - _id: 1, - overleaf: 1, - } - const options = {} - if (USE_QUERY_HINT) { - options.hint = { _id: 1 } - } - await batchedUpdate( - 'projects', - { - $and: [ - { 'overleaf.history.display': { $ne: true } }, - { 'overleaf.history.id': { $exists: true } }, - ], - }, - processBatch, - projection, - options - ) - console.log('Final') - console.log(RESULT) -} - -main() - .then(() => { - console.error('Done.') - process.exit(0) - }) - .catch(error => { - console.error({ error }) - process.exit(1) - }) diff --git a/services/web/scripts/recover_docs_from_redis.js b/services/web/scripts/recover_docs_from_redis.js index e18a3e7a0e..28614c6529 100644 --- a/services/web/scripts/recover_docs_from_redis.js +++ b/services/web/scripts/recover_docs_from_redis.js @@ -165,7 +165,6 @@ async function deleteDocFromRedis(projectId, docId) { `UnflushedTime:{${docId}}`, `Pathname:{${docId}}`, `ProjectHistoryId:{${docId}}`, - `ProjectHistoryType:{${docId}}`, `PendingUpdates:{${docId}}`, `lastUpdatedAt:{${docId}}`, `lastUpdatedBy:{${docId}}` diff --git a/services/web/test/acceptance/config/settings.test.defaults.js b/services/web/test/acceptance/config/settings.test.defaults.js index d871d52b23..a2288eea03 100644 --- a/services/web/test/acceptance/config/settings.test.defaults.js +++ b/services/web/test/acceptance/config/settings.test.defaults.js @@ -51,9 +51,6 @@ module.exports = { url: 'http://localhost:23005', host: 'localhost', }, - trackchanges: { - url: 'http://localhost:23015', - }, docstore: { url: 'http://localhost:23016', pubUrl: 'http://localhost:23016', @@ -78,8 +75,6 @@ module.exports = { }, project_history: { sendProjectStructureOps: true, - initializeHistoryForNewProjects: true, - displayHistoryForNewProjects: true, url: `http://localhost:23054`, }, v1_history: { diff --git a/services/web/test/unit/src/Documents/DocumentControllerTests.js b/services/web/test/unit/src/Documents/DocumentControllerTests.js index 44b01f8d4e..041dae0389 100644 --- a/services/web/test/unit/src/Documents/DocumentControllerTests.js +++ b/services/web/test/unit/src/Documents/DocumentControllerTests.js @@ -39,77 +39,6 @@ describe('DocumentController', function () { } }) - describe('when the project exists without project history enabled', function () { - beforeEach(function () { - this.project = { _id: this.project_id } - this.ProjectGetter.getProject = sinon - .stub() - .callsArgWith(2, null, this.project) - }) - - describe('when the document exists', function () { - beforeEach(function () { - this.doc = { _id: this.doc_id } - this.ProjectLocator.findElement = sinon - .stub() - .callsArgWith(1, null, this.doc, { fileSystem: this.pathname }) - this.ProjectEntityHandler.getDoc = sinon - .stub() - .yields(null, this.doc_lines, this.rev, this.version, this.ranges) - this.DocumentController.getDocument(this.req, this.res, this.next) - }) - - it('should get the project', function () { - this.ProjectGetter.getProject - .calledWith(this.project_id, { rootFolder: true, overleaf: true }) - .should.equal(true) - }) - - it('should get the pathname of the document', function () { - this.ProjectLocator.findElement - .calledWith({ - project: this.project, - element_id: this.doc_id, - type: 'doc', - }) - .should.equal(true) - }) - - it('should get the document content', function () { - this.ProjectEntityHandler.getDoc - .calledWith(this.project_id, this.doc_id) - .should.equal(true) - }) - - it('should return the document data to the client as JSON', function () { - this.res.type.should.equal('application/json') - this.res.body.should.equal( - JSON.stringify({ - lines: this.doc_lines, - version: this.version, - ranges: this.ranges, - pathname: this.pathname, - }) - ) - }) - }) - - describe("when the document doesn't exist", function () { - beforeEach(function () { - this.ProjectLocator.findElement = sinon - .stub() - .callsArgWith(1, new Errors.NotFoundError('not found')) - this.DocumentController.getDocument(this.req, this.res, this.next) - }) - - it('should call next with the NotFoundError', function () { - this.next - .calledWith(sinon.match.instanceOf(Errors.NotFoundError)) - .should.equal(true) - }) - }) - }) - describe('when project exists with project history enabled', function () { beforeEach(function () { this.doc = { _id: this.doc_id } @@ -152,53 +81,6 @@ describe('DocumentController', function () { }) }) - describe('when project exists that was migrated with downgrades allowed', function () { - beforeEach(function () { - this.doc = { _id: this.doc_id } - this.projectHistoryId = 1234 - this.projectHistoryDisplay = true - this.projectHistoryType = undefined - this.project = { - _id: this.project_id, - overleaf: { - history: { - id: this.projectHistoryId, - display: this.projectHistoryDisplay, - allowDowngrade: true, - }, - }, - } - this.ProjectGetter.getProject = sinon - .stub() - .callsArgWith(2, null, this.project) - this.ProjectLocator.findElement = sinon - .stub() - .callsArgWith(1, null, this.doc, { fileSystem: this.pathname }) - this.ProjectEntityHandler.getDoc = sinon - .stub() - .yields(null, this.doc_lines, this.rev, this.version, this.ranges) - return this.DocumentController.getDocument( - this.req, - this.res, - this.next - ) - }) - - it('should return the history id in the JSON but not history type, sending history to both services', function () { - this.res.type.should.equal('application/json') - return this.res.body.should.equal( - JSON.stringify({ - lines: this.doc_lines, - version: this.version, - ranges: this.ranges, - pathname: this.pathname, - projectHistoryId: this.projectHistoryId, - projectHistoryType: this.projectHistoryType, - }) - ) - }) - }) - describe('when the project does not exist', function () { beforeEach(function () { this.ProjectGetter.getProject = sinon.stub().callsArgWith(2, null, null) diff --git a/services/web/test/unit/src/History/HistoryControllerTests.js b/services/web/test/unit/src/History/HistoryControllerTests.js index 66b615dbc2..50fe94f32d 100644 --- a/services/web/test/unit/src/History/HistoryControllerTests.js +++ b/services/web/test/unit/src/History/HistoryControllerTests.js @@ -46,60 +46,12 @@ describe('HistoryController', function () { }, }) return (this.settings.apis = { - trackchanges: { - enabled: false, - url: 'http://trackchanges.example.com', - }, project_history: { url: 'http://project_history.example.com', }, }) }) - describe('selectHistoryApi', function () { - beforeEach(function () { - this.req = { url: '/mock/url', method: 'POST', params: {} } - this.res = 'mock-res' - return (this.next = sinon.stub()) - }) - - describe('for a project with project history', function () { - beforeEach(function () { - this.ProjectDetailsHandler.getDetails = sinon - .stub() - .callsArgWith(1, null, { - overleaf: { history: { id: 42, display: true } }, - }) - return this.HistoryController.selectHistoryApi( - this.req, - this.res, - this.next - ) - }) - - it('should set the flag for project history to true', function () { - return this.req.useProjectHistory.should.equal(true) - }) - }) - - describe('for any other project ', function () { - beforeEach(function () { - this.ProjectDetailsHandler.getDetails = sinon - .stub() - .callsArgWith(1, null, {}) - return this.HistoryController.selectHistoryApi( - this.req, - this.res, - this.next - ) - }) - - it('should not set the flag for project history to false', function () { - return this.req.useProjectHistory.should.equal(false) - }) - }) - }) - describe('proxyToHistoryApi', function () { beforeEach(function () { this.req = { url: '/mock/url', method: 'POST' } @@ -161,18 +113,6 @@ describe('HistoryController', function () { .should.equal(true) }) - it('should call the track changes api', function () { - return this.request - .calledWith({ - url: `${this.settings.apis.trackchanges.url}${this.req.url}`, - method: this.req.method, - headers: { - 'X-User-Id': this.user_id, - }, - }) - .should.equal(true) - }) - it('should pipe the response to the client', function () { expect(this.Stream.pipeline).to.have.been.calledWith( this.proxy, @@ -249,19 +189,6 @@ describe('HistoryController', function () { .should.equal(true) }) - it('should call the track changes api', function () { - return this.request - .calledWith({ - url: `${this.settings.apis.trackchanges.url}${this.req.url}`, - method: this.req.method, - json: true, - headers: { - 'X-User-Id': this.user_id, - }, - }) - .should.equal(true) - }) - it('should inject the user data', function () { return this.HistoryManager.injectUserDetails .calledWith(this.data) diff --git a/services/web/test/unit/src/History/HistoryManagerTests.js b/services/web/test/unit/src/History/HistoryManagerTests.js index 91d6180db8..b64662956d 100644 --- a/services/web/test/unit/src/History/HistoryManagerTests.js +++ b/services/web/test/unit/src/History/HistoryManagerTests.js @@ -23,10 +23,6 @@ describe('HistoryManager', function () { this.v1HistoryPassword = 'verysecret' this.settings = { apis: { - trackchanges: { - enabled: false, - url: 'http://trackchanges.example.com', - }, project_history: { url: this.projectHistoryUrl, }, @@ -55,56 +51,45 @@ describe('HistoryManager', function () { }) describe('initializeProject', function () { - describe('with project history enabled', function () { - beforeEach(function () { - this.settings.apis.project_history.initializeHistoryForNewProjects = true + beforeEach(function () { + this.settings.apis.project_history.initializeHistoryForNewProjects = true + }) + + describe('project history returns a successful response', function () { + beforeEach(async function () { + this.response.json.resolves({ project: { id: this.historyId } }) + this.result = await this.HistoryManager.promises.initializeProject( + this.historyId + ) }) - describe('project history returns a successful response', function () { - beforeEach(async function () { - this.response.json.resolves({ project: { id: this.historyId } }) - this.result = await this.HistoryManager.promises.initializeProject( - this.historyId - ) - }) - - it('should call the project history api', function () { - this.fetch.should.have.been.calledWithMatch( - `${this.settings.apis.project_history.url}/project`, - { method: 'POST' } - ) - }) - - it('should return the overleaf id', function () { - expect(this.result).to.equal(this.historyId) - }) + it('should call the project history api', function () { + this.fetch.should.have.been.calledWithMatch( + `${this.settings.apis.project_history.url}/project`, + { method: 'POST' } + ) }) - describe('project history returns a response without the project id', function () { - it('should throw an error', async function () { - this.response.json.resolves({ project: {} }) - await expect( - this.HistoryManager.promises.initializeProject(this.historyId) - ).to.be.rejected - }) - }) - - describe('project history errors', function () { - it('should propagate the error', async function () { - this.fetch.rejects(new Error('problem connecting')) - await expect( - this.HistoryManager.promises.initializeProject(this.historyId) - ).to.be.rejected - }) + it('should return the overleaf id', function () { + expect(this.result).to.equal(this.historyId) }) }) - describe('with project history disabled', function () { - it('should return without errors', async function () { - this.settings.apis.project_history.initializeHistoryForNewProjects = false + describe('project history returns a response without the project id', function () { + it('should throw an error', async function () { + this.response.json.resolves({ project: {} }) await expect( this.HistoryManager.promises.initializeProject(this.historyId) - ).to.be.fulfilled + ).to.be.rejected + }) + }) + + describe('project history errors', function () { + it('should propagate the error', async function () { + this.fetch.rejects(new Error('problem connecting')) + await expect( + this.HistoryManager.promises.initializeProject(this.historyId) + ).to.be.rejected }) }) })