Merge pull request #13327 from overleaf/msm-web-track-changes-cleanup

[web] Cleanup track-changes from web

GitOrigin-RevId: 8cef709ec5e91e4ffe8cd06826038ed84f36ef67

package-lock.json

@@ -379,6 +379,7 @@
       "version": "3.0.0"
     },
     "libraries/stream-utils": {
+      "name": "@overleaf/stream-utils",
       "version": "0.1.0",
       "license": "AGPL-3.0-only",
       "devDependencies": {
@@ -14620,18 +14621,6 @@
         "node": "*"
       }
     },
-    "node_modules/binary": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
-      "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=",
-      "dependencies": {
-        "buffers": "~0.1.1",
-        "chainsaw": "~0.1.0"
-      },
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/binary-extensions": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -15036,14 +15025,6 @@
       "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
       "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
     },
-    "node_modules/buffer-indexof-polyfill": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz",
-      "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==",
-      "engines": {
-        "node": ">=0.10"
-      }
-    },
     "node_modules/buffer-writer": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
@@ -15611,17 +15592,6 @@
         "node": ">=4"
       }
     },
-    "node_modules/chainsaw": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
-      "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=",
-      "dependencies": {
-        "traverse": ">=0.3.0 <0.4"
-      },
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/chalk": {
       "version": "2.4.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@@ -19247,41 +19217,6 @@
         "node": ">=0.10"
       }
     },
-    "node_modules/duplexer2": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz",
-      "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=",
-      "dependencies": {
-        "readable-stream": "^2.0.2"
-      }
-    },
-    "node_modules/duplexer2/node_modules/isarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
-    },
-    "node_modules/duplexer2/node_modules/readable-stream": {
-      "version": "2.3.7",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
-      "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
-      "dependencies": {
-        "core-util-is": "~1.0.0",
-        "inherits": "~2.0.3",
-        "isarray": "~1.0.0",
-        "process-nextick-args": "~2.0.0",
-        "safe-buffer": "~5.1.1",
-        "string_decoder": "~1.1.1",
-        "util-deprecate": "~1.0.1"
-      }
-    },
-    "node_modules/duplexer2/node_modules/string_decoder": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-      "dependencies": {
-        "safe-buffer": "~5.1.0"
-      }
-    },
     "node_modules/duplexify": {
       "version": "3.7.1",
       "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
@@ -22682,20 +22617,6 @@
         "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
       }
     },
-    "node_modules/fstream": {
-      "version": "1.0.12",
-      "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz",
-      "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==",
-      "dependencies": {
-        "graceful-fs": "^4.1.2",
-        "inherits": "~2.0.0",
-        "mkdirp": ">=0.5 0",
-        "rimraf": "2"
-      },
-      "engines": {
-        "node": ">=0.6"
-      }
-    },
     "node_modules/function-bind": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
@@ -27720,11 +27641,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/listenercount": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz",
-      "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc="
-    },
     "node_modules/listr2": {
       "version": "3.14.0",
       "resolved": "https://registry.npmjs.org/listr2/-/listr2-3.14.0.tgz",
@@ -35128,11 +35044,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/setimmediate": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
-      "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU="
-    },
     "node_modules/setprototypeof": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
@@ -37594,14 +37505,6 @@
         "punycode": "^2.1.0"
       }
     },
-    "node_modules/traverse": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
-      "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=",
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/ts-dedent": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.2.0.tgz",
@@ -38059,55 +37962,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/unzipper": {
-      "version": "0.10.11",
-      "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.11.tgz",
-      "integrity": "sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==",
-      "dependencies": {
-        "big-integer": "^1.6.17",
-        "binary": "~0.3.0",
-        "bluebird": "~3.4.1",
-        "buffer-indexof-polyfill": "~1.0.0",
-        "duplexer2": "~0.1.4",
-        "fstream": "^1.0.12",
-        "graceful-fs": "^4.2.2",
-        "listenercount": "~1.0.1",
-        "readable-stream": "~2.3.6",
-        "setimmediate": "~1.0.4"
-      }
-    },
-    "node_modules/unzipper/node_modules/bluebird": {
-      "version": "3.4.7",
-      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz",
-      "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM="
-    },
-    "node_modules/unzipper/node_modules/isarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
-    },
-    "node_modules/unzipper/node_modules/readable-stream": {
-      "version": "2.3.7",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
-      "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
-      "dependencies": {
-        "core-util-is": "~1.0.0",
-        "inherits": "~2.0.3",
-        "isarray": "~1.0.0",
-        "process-nextick-args": "~2.0.0",
-        "safe-buffer": "~5.1.1",
-        "string_decoder": "~1.1.1",
-        "util-deprecate": "~1.0.1"
-      }
-    },
-    "node_modules/unzipper/node_modules/string_decoder": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-      "dependencies": {
-        "safe-buffer": "~5.1.0"
-      }
-    },
     "node_modules/update-browserslist-db": {
       "version": "1.0.11",
       "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz",
@@ -41792,7 +41646,6 @@
         "scroll-into-view-if-needed": "^2.2.25",
         "tsscmp": "^1.0.6",
         "underscore": "^1.13.1",
-        "unzipper": "^0.10.11",
         "utf-8-validate": "^5.0.2",
         "uuid": "^3.0.1",
         "valid-data-url": "^2.0.0",
@@ -50588,7 +50441,6 @@
         "tsscmp": "^1.0.6",
         "typescript": "^4.5.5",
         "underscore": "^1.13.1",
-        "unzipper": "^0.10.11",
         "utf-8-validate": "^5.0.2",
         "uuid": "^3.0.1",
         "val-loader": "^5.0.1",
@@ -56100,15 +55952,6 @@
       "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.2.tgz",
       "integrity": "sha512-GAcQvbpsM0pUb0zw1EI0KhQEZ+lRwR5fYaAp3vPOYuP7aDvGy6cVN6XHLauvF8SOga2y0dcLcjt3iQDTSEliyw=="
     },
-    "binary": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
-      "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=",
-      "requires": {
-        "buffers": "~0.1.1",
-        "chainsaw": "~0.1.0"
-      }
-    },
     "binary-extensions": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -56444,11 +56287,6 @@
       "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
       "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
     },
-    "buffer-indexof-polyfill": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz",
-      "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A=="
-    },
     "buffer-writer": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
@@ -56881,14 +56719,6 @@
         "superagent": "^3.7.0"
       }
     },
-    "chainsaw": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
-      "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=",
-      "requires": {
-        "traverse": ">=0.3.0 <0.4"
-      }
-    },
     "chalk": {
       "version": "2.4.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@@ -59543,43 +59373,6 @@
         "nan": "^2.14.0"
       }
     },
-    "duplexer2": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz",
-      "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=",
-      "requires": {
-        "readable-stream": "^2.0.2"
-      },
-      "dependencies": {
-        "isarray": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
-        },
-        "readable-stream": {
-          "version": "2.3.7",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
-          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
-          "requires": {
-            "core-util-is": "~1.0.0",
-            "inherits": "~2.0.3",
-            "isarray": "~1.0.0",
-            "process-nextick-args": "~2.0.0",
-            "safe-buffer": "~5.1.1",
-            "string_decoder": "~1.1.1",
-            "util-deprecate": "~1.0.1"
-          }
-        },
-        "string_decoder": {
-          "version": "1.1.1",
-          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-          "requires": {
-            "safe-buffer": "~5.1.0"
-          }
-        }
-      }
-    },
     "duplexify": {
       "version": "3.7.1",
       "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
@@ -62206,17 +61999,6 @@
       "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
       "optional": true
     },
-    "fstream": {
-      "version": "1.0.12",
-      "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz",
-      "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==",
-      "requires": {
-        "graceful-fs": "^4.1.2",
-        "inherits": "~2.0.0",
-        "mkdirp": ">=0.5 0",
-        "rimraf": "2"
-      }
-    },
     "function-bind": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
@@ -66088,11 +65870,6 @@
         "repeat-string": "^1.5.2"
       }
     },
-    "listenercount": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz",
-      "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc="
-    },
     "listr2": {
       "version": "3.14.0",
       "resolved": "https://registry.npmjs.org/listr2/-/listr2-3.14.0.tgz",
@@ -72108,11 +71885,6 @@
         "to-object-path": "^0.3.0"
       }
     },
-    "setimmediate": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
-      "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU="
-    },
     "setprototypeof": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
@@ -74077,11 +73849,6 @@
         "punycode": "^2.1.0"
       }
     },
-    "traverse": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
-      "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk="
-    },
     "ts-dedent": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.2.0.tgz",
@@ -74423,57 +74190,6 @@
       "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz",
       "integrity": "sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw=="
     },
-    "unzipper": {
-      "version": "0.10.11",
-      "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.11.tgz",
-      "integrity": "sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==",
-      "requires": {
-        "big-integer": "^1.6.17",
-        "binary": "~0.3.0",
-        "bluebird": "~3.4.1",
-        "buffer-indexof-polyfill": "~1.0.0",
-        "duplexer2": "~0.1.4",
-        "fstream": "^1.0.12",
-        "graceful-fs": "^4.2.2",
-        "listenercount": "~1.0.1",
-        "readable-stream": "~2.3.6",
-        "setimmediate": "~1.0.4"
-      },
-      "dependencies": {
-        "bluebird": {
-          "version": "3.4.7",
-          "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz",
-          "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM="
-        },
-        "isarray": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
-        },
-        "readable-stream": {
-          "version": "2.3.7",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
-          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
-          "requires": {
-            "core-util-is": "~1.0.0",
-            "inherits": "~2.0.3",
-            "isarray": "~1.0.0",
-            "process-nextick-args": "~2.0.0",
-            "safe-buffer": "~5.1.1",
-            "string_decoder": "~1.1.1",
-            "util-deprecate": "~1.0.1"
-          }
-        },
-        "string_decoder": {
-          "version": "1.1.1",
-          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-          "requires": {
-            "safe-buffer": "~5.1.0"
-          }
-        }
-      }
-    },
     "update-browserslist-db": {
       "version": "1.0.11",
       "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz",

services/web/.gitignore

@@ -88,9 +88,6 @@ cypress/videos/
 cypress/downloads/
 cypress/results/
 
-# Test fixture zip
-!modules/history-migration/test/unit/src/data/track-changes-project.zip
-
 # Ace themes for conversion
 frontend/js/features/source-editor/themes/ace/
 

services/web/app/src/Features/Documents/DocumentController.js

@@ -51,21 +51,11 @@ function getDocument(req, res, next) {
                 plainTextResponse(res, lines.join('\n'))
               } else {
                 const projectHistoryId = _.get(project, 'overleaf.history.id')
-                const projectHistoryDisplay = _.get(
-                  project,
-                  'overleaf.history.display'
-                )
-                const sendToBothHistorySystems = _.get(
-                  project,
-                  'overleaf.history.allowDowngrade'
-                )
-                // if project has been switched but has 'allowDowngrade' set
-                // then leave projectHistoryType undefined to (temporarily)
-                // continue sending updates to both SL and full project history
-                const projectHistoryType =
-                  projectHistoryDisplay && !sendToBothHistorySystems
-                    ? 'project-history'
-                    : undefined // for backwards compatibility, don't send anything if the project is still on track-changes
+
+                // all projects are now migrated to Full Project History, keeping the field
+                // for API compatibility
+                const projectHistoryType = 'project-history'
+
                 res.json({
                   lines,
                   version,

services/web/app/src/Features/History/HistoryController.js

@@ -16,35 +16,9 @@ const { prepareZipAttachment } = require('../../infrastructure/Response')
 const Features = require('../../infrastructure/Features')
 
 module.exports = HistoryController = {
-  selectHistoryApi(req, res, next) {
-    const { Project_id: projectId } = req.params
-    // find out which type of history service this project uses
-    ProjectDetailsHandler.getDetails(projectId, function (err, project) {
-      if (err) {
-        return next(err)
-      }
-      const history = project.overleaf && project.overleaf.history
-      if (history && history.id && history.display) {
-        req.useProjectHistory = true
-      } else {
-        req.useProjectHistory = false
-      }
-      next()
-    })
-  },
-
-  ensureProjectHistoryEnabled(req, res, next) {
-    if (req.useProjectHistory) {
-      next()
-    } else {
-      res.sendStatus(404)
-    }
-  },
-
   proxyToHistoryApi(req, res, next) {
     const userId = SessionManager.getLoggedInUserId(req.session)
-    const url =
-      HistoryController.buildHistoryServiceUrl(req.useProjectHistory) + req.url
+    const url = settings.apis.project_history.url + req.url
 
     const getReq = request({
       url,
@@ -65,8 +39,7 @@ module.exports = HistoryController = {
 
   proxyToHistoryApiAndInjectUserDetails(req, res, next) {
     const userId = SessionManager.getLoggedInUserId(req.session)
-    const url =
-      HistoryController.buildHistoryServiceUrl(req.useProjectHistory) + req.url
+    const url = settings.apis.project_history.url + req.url
     HistoryController._makeRequest(
       {
         url,
@@ -90,16 +63,6 @@ module.exports = HistoryController = {
     )
   },
 
-  buildHistoryServiceUrl(useProjectHistory) {
-    // choose a history service, either document-level (trackchanges)
-    // or project-level (project_history)
-    if (useProjectHistory) {
-      return settings.apis.project_history.url
-    } else {
-      return settings.apis.trackchanges.url
-    }
-  },
-
   resyncProjectHistory(req, res, next) {
     // increase timeout to 6 minutes
     res.setTimeout(6 * 60 * 1000)

services/web/app/src/Features/History/HistoryManager.js

@@ -5,14 +5,6 @@ const OError = require('@overleaf/o-error')
 const UserGetter = require('../User/UserGetter')
 
 async function initializeProject(projectId) {
-  if (
-    !(
-      settings.apis.project_history &&
-      settings.apis.project_history.initializeHistoryForNewProjects
-    )
-  ) {
-    return null
-  }
   const response = await fetch(`${settings.apis.project_history.url}/project`, {
     method: 'POST',
     headers: {

services/web/app/src/Features/Project/ProjectController.js

@@ -1064,10 +1064,6 @@ const ProjectController = {
               editorThemes: THEME_LIST,
               legacyEditorThemes: LEGACY_THEME_LIST,
               maxDocLength: Settings.max_doc_length,
-              useV2History:
-                project.overleaf &&
-                project.overleaf.history &&
-                Boolean(project.overleaf.history.display),
               brandVariation,
               allowedImageNames,
               gitBridgePublicBaseUrl: Settings.gitBridgePublicBaseUrl,

services/web/app/src/Features/Project/ProjectCreationHandler.js

@@ -169,15 +169,10 @@ async function _createBlankProject(
     }
   }
 
-  // only display full project history when the project has the overleaf history id attribute
-  // (to allow scripted creation of projects without full project history)
-  const historyId = project.overleaf.history.id
-  if (
-    Settings.apis.project_history.displayHistoryForNewProjects &&
-    historyId != null
-  ) {
-    project.overleaf.history.display = true
-  }
+  // All the projects are initialised with Full Project History. This property
+  // is still set for backwards compatibility: Server Pro requires all projects
+  // have it set to `true` since SP 4.0
+  project.overleaf.history.display = true
 
   if (Settings.currentImageName) {
     // avoid clobbering any imageName already set in attributes (e.g. importedImageName)

services/web/app/src/Features/Project/ProjectHistoryHandler.js

@@ -35,73 +35,6 @@ const ProjectHistoryHandler = {
     })
   },
 
-  unsetHistory(projectId, callback) {
-    Project.updateOne(
-      { _id: projectId },
-      { $unset: { 'overleaf.history': true } },
-      callback
-    )
-  },
-
-  upgradeHistory(projectId, allowDowngrade, callback) {
-    // project must have an overleaf.history.id before allowing display of new history
-    Project.updateOne(
-      { _id: projectId, 'overleaf.history.id': { $exists: true } },
-      {
-        'overleaf.history.display': true,
-        'overleaf.history.upgradedAt': new Date(),
-        'overleaf.history.allowDowngrade': allowDowngrade,
-      },
-      function (err, result) {
-        if (err) {
-          return callback(err)
-        }
-        // return an error if overleaf.history.id wasn't present
-        if (result.matchedCount === 0) {
-          return callback(new Error('history not upgraded'))
-        }
-        callback()
-      }
-    )
-  },
-
-  downgradeHistory(projectId, callback) {
-    Project.updateOne(
-      { _id: projectId, 'overleaf.history.upgradedAt': { $exists: true } },
-      {
-        'overleaf.history.display': false,
-        $unset: { 'overleaf.history.upgradedAt': 1 },
-      },
-      function (err, result) {
-        if (err) {
-          return callback(err)
-        }
-        if (result.matchedCount === 0) {
-          return callback(new Error('history not downgraded'))
-        }
-        callback()
-      }
-    )
-  },
-
-  setMigrationArchiveFlag(projectId, callback) {
-    Project.updateOne(
-      { _id: projectId, version: { $exists: true } },
-      {
-        'overleaf.history.zipFileArchivedInProject': true,
-      },
-      function (err, result) {
-        if (err) {
-          return callback(err)
-        }
-        if (result.matchedCount === 0) {
-          return callback(new Error('migration flag not set'))
-        }
-        callback()
-      }
-    )
-  },
-
   ensureHistoryExistsForProject(projectId, callback) {
     // We can only set a history id for a project that doesn't have one. The
     // history id is cached in the project history service, and changing an

services/web/app/src/router.js

@@ -711,34 +711,29 @@ function initialize(webRouter, privateApiRouter, publicApiRouter) {
     '/project/:Project_id/updates',
     AuthorizationMiddleware.blockRestrictedUserFromProject,
     AuthorizationMiddleware.ensureUserCanReadProject,
-    HistoryController.selectHistoryApi,
     HistoryController.proxyToHistoryApiAndInjectUserDetails
   )
   webRouter.get(
     '/project/:Project_id/doc/:doc_id/diff',
     AuthorizationMiddleware.blockRestrictedUserFromProject,
     AuthorizationMiddleware.ensureUserCanReadProject,
-    HistoryController.selectHistoryApi,
     HistoryController.proxyToHistoryApi
   )
   webRouter.get(
     '/project/:Project_id/diff',
     AuthorizationMiddleware.blockRestrictedUserFromProject,
     AuthorizationMiddleware.ensureUserCanReadProject,
-    HistoryController.selectHistoryApi,
     HistoryController.proxyToHistoryApiAndInjectUserDetails
   )
   webRouter.get(
     '/project/:Project_id/filetree/diff',
     AuthorizationMiddleware.blockRestrictedUserFromProject,
     AuthorizationMiddleware.ensureUserCanReadProject,
-    HistoryController.selectHistoryApi,
     HistoryController.proxyToHistoryApi
   )
   webRouter.post(
     '/project/:Project_id/doc/:doc_id/version/:version_id/restore',
     AuthorizationMiddleware.ensureUserCanWriteProjectContent,
-    HistoryController.selectHistoryApi,
     HistoryController.proxyToHistoryApi
   )
   webRouter.post(
@@ -768,22 +763,16 @@ function initialize(webRouter, privateApiRouter, publicApiRouter) {
     '/project/:Project_id/labels',
     AuthorizationMiddleware.blockRestrictedUserFromProject,
     AuthorizationMiddleware.ensureUserCanReadProject,
-    HistoryController.selectHistoryApi,
-    HistoryController.ensureProjectHistoryEnabled,
     HistoryController.getLabels
   )
   webRouter.post(
     '/project/:Project_id/labels',
     AuthorizationMiddleware.ensureUserCanWriteProjectContent,
-    HistoryController.selectHistoryApi,
-    HistoryController.ensureProjectHistoryEnabled,
     HistoryController.createLabel
   )
   webRouter.delete(
     '/project/:Project_id/labels/:label_id',
     AuthorizationMiddleware.ensureUserCanWriteProjectContent,
-    HistoryController.selectHistoryApi,
-    HistoryController.ensureProjectHistoryEnabled,
     HistoryController.deleteLabel
   )
 

services/web/app/views/project/editor/meta.pug

@@ -1,4 +1,3 @@
-meta(name="ol-useV2History" data-type="boolean" content=useV2History)
 meta(name="ol-project_id" content=project_id)
 meta(name="ol-projectName" content=projectName)
 meta(name="ol-userSettings" data-type="json" content=userSettings)

services/web/config/settings.defaults.js

@@ -186,9 +186,6 @@ module.exports = {
       url: `http://${process.env.SPELLING_HOST || 'localhost'}:3005`,
       host: process.env.SPELLING_HOST,
     },
-    trackchanges: {
-      url: `http://${process.env.TRACK_CHANGES_HOST || 'localhost'}:3015`,
-    },
     docstore: {
       url: `http://${process.env.DOCSTORE_HOST || 'localhost'}:3016`,
       pubUrl: `http://${process.env.DOCSTORE_HOST || 'localhost'}:3016`,
@@ -207,8 +204,6 @@ module.exports = {
     },
     project_history: {
       sendProjectStructureOps: true,
-      initializeHistoryForNewProjects: true,
-      displayHistoryForNewProjects: true,
       url: `http://${process.env.PROJECT_HISTORY_HOST || 'localhost'}:3054`,
     },
     realTime: {
@@ -810,12 +805,7 @@ module.exports = {
     oauth2Server: [],
   },
 
-  moduleImportSequence: [
-    'launchpad',
-    'server-ce-scripts',
-    'user-activate',
-    'history-migration',
-  ],
+  moduleImportSequence: ['launchpad', 'server-ce-scripts', 'user-activate'],
 
   csp: {
     enabled: process.env.CSP_ENABLED === 'true',

services/web/frontend/js/ide.js

@@ -22,7 +22,6 @@ import LoadingManager from './ide/LoadingManager'
 import ConnectionManager from './ide/connection/ConnectionManager'
 import EditorManager from './ide/editor/EditorManager'
 import OnlineUsersManager from './ide/online-users/OnlineUsersManager'
-import HistoryManager from './ide/history/HistoryManager'
 import HistoryV2Manager from './ide/history/HistoryV2Manager'
 import PermissionsManager from './ide/permissions/PermissionsManager'
 import BinaryFilesManager from './ide/binary-files/BinaryFilesManager'
@@ -213,11 +212,7 @@ App.controller(
       eventTracking
     )
     ide.onlineUsersManager = new OnlineUsersManager(ide, $scope)
-    if (window.data.useV2History) {
-      ide.historyManager = new HistoryV2Manager(ide, $scope, localStorage)
-    } else {
-      ide.historyManager = new HistoryManager(ide, $scope)
-    }
+    ide.historyManager = new HistoryV2Manager(ide, $scope, localStorage)
     ide.permissionsManager = new PermissionsManager(ide, $scope)
     ide.binaryFilesManager = new BinaryFilesManager(ide, $scope)
     ide.metadataManager = new MetadataManager(ide, $scope, metadata)

services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js

@@ -1,394 +0,0 @@
-const { ObjectId } = require('mongodb')
-const {
-  db,
-  READ_PREFERENCE_SECONDARY,
-} = require('../../../../app/src/infrastructure/mongodb')
-const Settings = require('@overleaf/settings')
-
-const ProjectHistoryHandler = require('../../../../app/src/Features/Project/ProjectHistoryHandler')
-const HistoryManager = require('../../../../app/src/Features/History/HistoryManager')
-const ProjectHistoryController = require('./ProjectHistoryController')
-const ProjectEntityHandler = require('../../../../app/src/Features/Project/ProjectEntityHandler')
-const ProjectEntityUpdateHandler = require('../../../../app/src/Features/Project/ProjectEntityUpdateHandler')
-const DocumentUpdaterHandler = require('../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler')
-
-// Timestamp of when 'Enable history for SL in background' release
-const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED =
-  Settings.apis.project_history?.idWhenFullProjectHistoryEnabled // was '5a8d8a370000000000000000'
-const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED =
-  ID_WHEN_FULL_PROJECT_HISTORY_ENABLED
-    ? new ObjectId(ID_WHEN_FULL_PROJECT_HISTORY_ENABLED).getTimestamp()
-    : null
-
-async function countProjects(query = {}) {
-  const count = await db.projects.countDocuments(query)
-  return count
-}
-
-async function countDocHistory(query = {}) {
-  const count = await db.docHistory.countDocuments(query)
-  return count
-}
-
-async function findProjects(query = {}, projection = {}) {
-  const projects = await db.projects.find(query).project(projection).toArray()
-  return projects
-}
-
-async function determineProjectHistoryType(project) {
-  if (project.overleaf && project.overleaf.history) {
-    if (project.overleaf.history.upgradeFailed) {
-      return 'UpgradeFailed'
-    }
-    if (project.overleaf.history.conversionFailed) {
-      return 'ConversionFailed'
-    }
-  }
-  if (
-    project.overleaf &&
-    project.overleaf.history &&
-    project.overleaf.history.id
-  ) {
-    if (project.overleaf.history.display) {
-      // v2: full project history, do nothing
-      return 'V2'
-    } else {
-      if (projectCreatedAfterFullProjectHistoryEnabled(project)) {
-        // IF project initialised after full project history enabled for all projects
-        //    THEN project history should contain all information we need, without intervention
-        return 'V1WithoutConversion'
-      } else {
-        // ELSE SL history may predate full project history
-        //    THEN delete full project history and convert their SL history to full project history
-        // --
-        // TODO: how to verify this, can get rough start date of SL history, but not full project history
-        const preserveHistory = await shouldPreserveHistory(project)
-        const anyDocHistory = await anyDocHistoryExists(project)
-        const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-        if (preserveHistory) {
-          if (anyDocHistory || anyDocHistoryIndex) {
-            // if SL history exists that we need to preserve, then we must convert
-            return 'V1WithConversion'
-          } else {
-            // otherwise just upgrade without conversion
-            return 'V1WithoutConversion'
-          }
-        } else {
-          // if preserveHistory false, then max 7 days of SL history
-          // but v1 already record to both histories, so safe to upgrade
-          return 'V1WithoutConversion'
-        }
-      }
-    }
-  } else {
-    const preserveHistory = await shouldPreserveHistory(project)
-    const anyDocHistory = await anyDocHistoryExists(project)
-    const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-    if (anyDocHistory || anyDocHistoryIndex) {
-      // IF there is SL history ->
-      if (preserveHistory) {
-        // that needs to be preserved:
-        //    THEN initialise full project history and convert SL history to full project history
-        return 'NoneWithConversion'
-      } else {
-        return 'NoneWithTemporaryHistory'
-      }
-    } else {
-      // ELSE there is not any SL history ->
-      //    THEN initialise full project history and sync with current content
-      return 'NoneWithoutConversion'
-    }
-  }
-}
-
-async function upgradeProject(project, options) {
-  const historyType = await determineProjectHistoryType(project)
-  if (historyType === 'V2') {
-    return { historyType, upgraded: true }
-  }
-  const upgradeFn = getUpgradeFunctionForType(historyType)
-  if (!upgradeFn) {
-    return { error: 'unsupported history type' }
-  }
-  if (options.forceClean) {
-    try {
-      const projectId = project._id
-      // delete any existing history stored in the mongo backend
-      await HistoryManager.promises.deleteProject(projectId, projectId)
-      // unset overleaf.history.id to prevent the migration script from failing on checks
-      await db.projects.updateOne(
-        { _id: projectId },
-        { $unset: { 'overleaf.history.id': '' } }
-      )
-    } catch (err) {
-      // failed to delete existing history, but we can try to continue
-    }
-  }
-  const result = await upgradeFn(project, options)
-  result.historyType = historyType
-  return result
-}
-
-// Do upgrades/conversion:
-
-function getUpgradeFunctionForType(historyType) {
-  return UpgradeFunctionMapping[historyType]
-}
-
-const UpgradeFunctionMapping = {
-  NoneWithoutConversion: doUpgradeForNoneWithoutConversion,
-  UpgradeFailed: doUpgradeForNoneWithoutConversion,
-  ConversionFailed: doUpgradeForNoneWithConversion,
-  V1WithoutConversion: doUpgradeForV1WithoutConversion,
-  V1WithConversion: doUpgradeForV1WithConversion,
-  NoneWithConversion: doUpgradeForNoneWithConversion,
-  NoneWithTemporaryHistory: doUpgradeForNoneWithConversion,
-}
-
-async function doUpgradeForV1WithoutConversion(project) {
-  await db.projects.updateOne(
-    { _id: project._id },
-    {
-      $set: {
-        'overleaf.history.display': true,
-        'overleaf.history.upgradedAt': new Date(),
-        'overleaf.history.upgradeReason': `v1-without-sl-history`,
-      },
-    }
-  )
-  return { upgraded: true }
-}
-
-async function doUpgradeForV1WithConversion(project) {
-  const result = {}
-  const projectId = project._id
-  // migrateProjectHistory expects project id as a string
-  const projectIdString = project._id.toString()
-  try {
-    // We treat these essentially as None projects, the V1 history is irrelevant,
-    // so we will delete it, and do a conversion as if we're a None project
-    await ProjectHistoryController.deleteProjectHistory(projectIdString)
-    await ProjectHistoryController.migrateProjectHistory(projectIdString)
-  } catch (err) {
-    // if migrateProjectHistory fails, it cleans up by deleting
-    // the history and unsetting the history id
-    // therefore a failed project will still look like a 'None with conversion' project
-    result.error = err
-    await db.projects.updateOne(
-      { _id: projectId },
-      {
-        $set: {
-          'overleaf.history.conversionFailed': true,
-        },
-      }
-    )
-    return result
-  }
-  await db.projects.updateOne(
-    { _id: projectId },
-    {
-      $set: {
-        'overleaf.history.upgradeReason': `v1-with-conversion`,
-      },
-      $unset: {
-        'overleaf.history.upgradeFailed': true,
-        'overleaf.history.conversionFailed': true,
-      },
-    }
-  )
-  result.upgraded = true
-  return result
-}
-
-async function doUpgradeForNoneWithoutConversion(project) {
-  const result = {}
-  const projectId = project._id
-  try {
-    // Logic originally from ProjectHistoryHandler.ensureHistoryExistsForProject
-    // However sends a force resync project to project history instead
-    // of a resync request to doc-updater
-    let historyId = await ProjectHistoryHandler.promises.getHistoryId(projectId)
-    if (historyId == null) {
-      historyId = await HistoryManager.promises.initializeProject(projectId)
-      if (historyId != null) {
-        await ProjectHistoryHandler.promises.setHistoryId(projectId, historyId)
-      }
-    }
-    // tell document updater to clear the docs, they will be reloaded with any new history id
-    await DocumentUpdaterHandler.promises.flushProjectToMongoAndDelete(
-      projectId
-    )
-    // now resync the project
-    await HistoryManager.promises.resyncProject(projectId, {
-      force: true,
-      origin: { kind: 'history-migration' },
-    })
-    await HistoryManager.promises.flushProject(projectId)
-  } catch (err) {
-    result.error = err
-    await db.projects.updateOne(
-      { _id: project._id },
-      {
-        $set: {
-          'overleaf.history.upgradeFailed': true,
-        },
-      }
-    )
-    return result
-  }
-  await db.projects.updateOne(
-    { _id: project._id },
-    {
-      $set: {
-        'overleaf.history.display': true,
-        'overleaf.history.upgradedAt': new Date(),
-        'overleaf.history.upgradeReason': `none-without-conversion`,
-      },
-    }
-  )
-  result.upgraded = true
-  return result
-}
-
-async function doUpgradeForNoneWithConversion(project, options = {}) {
-  const result = {}
-  const projectId = project._id
-  // migrateProjectHistory expects project id as a string
-  const projectIdString = project._id.toString()
-  try {
-    if (options.convertLargeDocsToFile) {
-      result.convertedDocCount = await convertLargeDocsToFile(
-        projectId,
-        options.userId
-      )
-    }
-    await ProjectHistoryController.migrateProjectHistory(
-      projectIdString,
-      options.migrationOptions
-    )
-  } catch (err) {
-    // if migrateProjectHistory fails, it cleans up by deleting
-    // the history and unsetting the history id
-    // therefore a failed project will still look like a 'None with conversion' project
-    result.error = err
-    // We set a failed flag so future runs of the script don't automatically retry
-    await db.projects.updateOne(
-      { _id: projectId },
-      {
-        $set: {
-          'overleaf.history.conversionFailed': true,
-        },
-      }
-    )
-    return result
-  }
-  await db.projects.updateOne(
-    { _id: projectId },
-    {
-      $set: {
-        'overleaf.history.upgradeReason':
-          `none-with-conversion` + options.reason ? `/${options.reason}` : ``,
-      },
-      $unset: {
-        'overleaf.history.upgradeFailed': true,
-        'overleaf.history.conversionFailed': true,
-      },
-    }
-  )
-  result.upgraded = true
-  return result
-}
-
-// Util
-
-function projectCreatedAfterFullProjectHistoryEnabled(project) {
-  if (DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED == null) {
-    return false
-  } else {
-    return (
-      project._id.getTimestamp() >= DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED
-    )
-  }
-}
-
-async function shouldPreserveHistory(project) {
-  return await db.projectHistoryMetaData.findOne(
-    {
-      $and: [
-        { project_id: { $eq: project._id } },
-        { preserveHistory: { $eq: true } },
-      ],
-    },
-    { readPreference: READ_PREFERENCE_SECONDARY }
-  )
-}
-
-async function anyDocHistoryExists(project) {
-  return await db.docHistory.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: READ_PREFERENCE_SECONDARY,
-    }
-  )
-}
-
-async function anyDocHistoryIndexExists(project) {
-  return await db.docHistoryIndex.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: READ_PREFERENCE_SECONDARY,
-    }
-  )
-}
-
-async function convertLargeDocsToFile(projectId, userId) {
-  const docs = await ProjectEntityHandler.promises.getAllDocs(projectId)
-  let convertedDocCount = 0
-  for (const doc of Object.values(docs)) {
-    const sizeBound = JSON.stringify(doc.lines)
-    if (docIsTooLarge(sizeBound, doc.lines, Settings.max_doc_length)) {
-      await ProjectEntityUpdateHandler.promises.convertDocToFile(
-        projectId,
-        doc._id,
-        userId,
-        null
-      )
-      convertedDocCount++
-    }
-  }
-  return convertedDocCount
-}
-
-// check whether the total size of the document in characters exceeds the
-// maxDocLength.
-//
-// Copied from document-updater:
-// https://github.com/overleaf/internal/blob/74adfbebda5f3c2c37d9937f0db5c4106ecde492/services/document-updater/app/js/Limits.js#L18
-function docIsTooLarge(estimatedSize, lines, maxDocLength) {
-  if (estimatedSize <= maxDocLength) {
-    return false // definitely under the limit, no need to calculate the total size
-  }
-  // calculate the total size, bailing out early if the size limit is reached
-  let size = 0
-  for (const line of lines) {
-    size += line.length + 1 // include the newline
-    if (size > maxDocLength) return true
-  }
-  // since we didn't hit the limit in the loop, the document is within the allowed length
-  return false
-}
-
-module.exports = {
-  countProjects,
-  countDocHistory,
-  findProjects,
-  determineProjectHistoryType,
-  getUpgradeFunctionForType,
-  upgradeProject,
-  convertLargeDocsToFile,
-  anyDocHistoryExists,
-  anyDocHistoryIndexExists,
-  doUpgradeForNoneWithConversion,
-}

services/web/modules/history-migration/app/src/ProjectHistoryController.js

@@ -1,1324 +0,0 @@
-const _ = require('lodash')
-const settings = require('@overleaf/settings')
-const OError = require('@overleaf/o-error')
-const fs = require('fs')
-const fse = require('fs-extra')
-const { ObjectId } = require('mongodb')
-const request = require('request')
-const { pipeline } = require('stream')
-const unzipper = require('unzipper')
-const util = require('util')
-const logger = require('@overleaf/logger')
-const path = require('path')
-const {
-  FileTooLargeError,
-  InvalidNameError,
-} = require('../../../../app/src/Features/Errors/Errors')
-const FilestoreHandler = require('../../../../app/src/Features/FileStore/FileStoreHandler')
-const ProjectGetter = require('../../../../app/src/Features/Project/ProjectGetter')
-const RedisWrapper = require('../../../../app/src/infrastructure/RedisWrapper')
-const HistoryManager = require('../../../../app/src/Features/History/HistoryManager')
-const ProjectHistoryHandler = require('../../../../app/src/Features/Project/ProjectHistoryHandler')
-const ProjectUpdateHandler = require('../../../../app/src/Features/Project/ProjectUpdateHandler')
-const DocumentUpdaterHandler = require('../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler')
-const ProjectEntityHandler = require('../../../../app/src/Features/Project/ProjectEntityHandler')
-const ProjectEntityUpdateHandler = require('../../../../app/src/Features/Project/ProjectEntityUpdateHandler')
-const SafePath = require('../../../../app/src/Features/Project/SafePath')
-const { DeletedFile } = require('../../../../app/src/models/DeletedFile')
-const { Doc } = require('../../../../app/src/models/Doc')
-const {
-  iterablePaths,
-} = require('../../../../app/src/Features/Project/IterablePath')
-
-const rclient = RedisWrapper.client('project_history_migration')
-
-module.exports = { deleteProjectHistory, migrateProjectHistory }
-
-/**
- * @typedef {Object} UpdateMeta
- * @property {string | null} user_id the id of the user that performed the update
- * @property {number} ts the timestamp of the update
- */
-
-/**
- * @typedef {UpdateMeta} EditDocUpdateMeta
- * @property {string | null} user_id
- * @property {number} ts
- * @property {string} pathname the doc pathname
- * @property {number} doc_length the length of the doc
- */
-
-/**
- * @typedef {Object} Update
- * @property {string} pathname the path in the file tree
- * @property {UpdateMeta} meta
- // * @property {string} version a two-part version. The first part is the project version after the updates, as recorded in Mongo. The second part is a counter that increments for each update in this batch.
- * @property {string} projectHistoryId the v1 history id for this project
- * @property {number} v
- */
-
-/**
- * @typedef {Update} FileUpdate
- * @property {string} pathname
- * @property {UpdateMeta} meta
- * @property {string} projectHistoryId
- * @property {number} v
- * @property {string} file
- */
-
-/**
- * @typedef {FileUpdate} AddFileUpdate
- * @property {string} pathname
- * @property {UpdateMeta} meta
- * @property {string} projectHistoryId
- * @property {number} v
- * @property {string} file
- * @property {string} url
- */
-
-/**
- * @typedef {Update} DocUpdate
- * @property {UpdateMeta} meta
- * @property {string} projectHistoryId
- * @property {number} v
- * @property {string} doc
- */
-
-/**
- * @typedef {DocUpdate} AddDocUpdate
- * @property {string} pathname
- * @property {UpdateMeta} meta
- * @property {string} projectHistoryId
- * @property {number} v
- * @property {string} doc
- * @property {string} docLines
- * @property {string} docLinesId
- * @property {boolean} contentStored
- */
-
-/**
- * @typedef {DocUpdate} EditDocUpdate
- * @property {EditDocUpdateMeta} meta
- * @property {string} projectHistoryId
- * @property {number} v
- * @property {number} lastV
- * @property {string} doc
- * @property {Array<Object>} op
- */
-
-/**
- * @typedef {AddDocUpdate | AddFileUpdate} AddUpdate
- */
-
-/**
- * @typedef {DocUpdate | FileUpdate} DeleteUpdate
- * @property {string} pathname
- * @property {UpdateMeta} meta
- * @property {string} projectHistoryId
- * @property {number} v
- * @property {string} doc
- * @property {string} new_pathname
- */
-
-/**
- * @typedef {Update} EditDocUpdateStub
- * @property {true} stub
- * @property {string} path
- * @property {string} pathname
- * @property {number} v
- * @property {number} doc_length
- */
-
-/**
- * @typedef {AddUpdate | DeleteUpdate | EditDocUpdate | EditDocUpdateStub } AnyUpdate
- */
-
-/**
- * @typedef {Object} Project
- * @property {string} _id the id of the user that performed the update
- * @property {Object} overleaf
- */
-
-/**
- * @typedef ManifestUpdate
- * @property {string} path
- * @property {number} doc_length
- * @property {number} ts
- * @property {number} version
- */
-
-/**
- * @typedef ManifestContent
- * @property {number} start
- */
-
-/**
- * @typedef ManifestDoc
- * @property {string} id
- * @property {ManifestContent} content
- * @property {Array<ManifestUpdate>} updates
- */
-
-/**
- * @typedef {Object} Manifest
- * @property {string} projectId
- * @property {Array<ManifestDoc>} docs
- */
-
-/**
- * @typedef Entity
- * @property {string} type
- * @property {string} path
- * @property {string} docLines
- * @property {string} deletedAt
- * @property {boolean} deleted
- */
-
-/**
- * Iterate recursively through the folders in project.rootFolder,
- * building a map of all the docs (with content as a docLines string)
- * and files (with content as a filestore URL).
- *
- * @param {Object} project
- * @returns {Promise<Map<string, Entity>>}
- */
-async function processRootFolder(project) {
-  const entities = new Map()
-
-  async function processFolder(folder, root = '') {
-    for (const item of iterablePaths(folder, 'docs')) {
-      const doc = await Doc.findOne(
-        item._id,
-        // only read the fields we need to save memory
-        { _id: 1, inS3: 1, lines: 1, name: 1 }
-      ).lean()
-
-      // skip malformed doc entries
-      if (!doc?._id) {
-        logger.warn({ doc }, 'skipping doc with missing id')
-        continue
-      }
-      const id = doc._id.toString()
-      const docIsInS3 = !!doc.inS3
-      let docLines
-
-      if (docIsInS3) {
-        const docPeek = await ProjectEntityHandler.promises.getDoc(
-          project._id,
-          item._id,
-          { peek: true }
-        )
-        docLines = docPeek.lines
-      } else {
-        docLines = doc.lines
-      }
-
-      if (!docLines) {
-        throw new Error(`no doc lines for doc ${id} (inS3: ${docIsInS3})`)
-      }
-
-      entities.set(id, {
-        path: `${root}/${item.name}`, // NOTE: not doc.name, which is "new doc",
-        type: 'doc',
-        docLines: docLines.join('\n'),
-      })
-    }
-
-    for (const item of iterablePaths(folder, 'fileRefs')) {
-      const path = `${root}/${item.name}`
-
-      // skip malformed file entries
-      if (!item?._id) {
-        logger.warn({ item }, 'skipping fileRef with missing id')
-        continue
-      }
-      const id = item._id.toString()
-
-      entities.set(id, {
-        path,
-        type: 'file',
-        url: FilestoreHandler._buildUrl(project._id.toString(), id),
-      })
-    }
-
-    for (const subfolder of iterablePaths(folder, 'folders')) {
-      const path = `${root}/${subfolder.name}`
-      await processFolder(subfolder, path)
-    }
-  }
-
-  for (const folder of project.rootFolder) {
-    await processFolder(folder)
-  }
-
-  return entities
-}
-
-/**
- * Read docs deleted from a project, from the Doc collection,
- * and add them to the entities map with the content in a docLines string.
- *
- * These entities have a `deleted` property set to `true` and a `deletedAt` date.
- *
- * @param {Map<string, Object>} entities
- * @param {string} projectId
- * @returns {Promise<void>}
- */
-async function readDeletedDocs(entities, projectId) {
-  // NOTE: could call DocstoreManager.promises.getAllDeletedDocs(projectId) instead
-
-  // Look for all docs, since some deleted docs are found in track-changes manifest,
-  // but do not have deleted flag set for reasons that are unclear
-  // (we will not add docs to entities if they were previously added by processRootFolder)
-  const deletedDocsCursor = Doc.find(
-    {
-      project_id: ObjectId(projectId),
-    },
-    // only read the fields we need to save memory
-    { _id: 1, inS3: 1, lines: 1, name: 1, deletedAt: 1 }
-  )
-    .lean()
-    .cursor()
-  for await (const doc of deletedDocsCursor) {
-    // skip malformed deleted doc entries
-    if (!doc?._id) {
-      logger.warn({ doc }, 'skipping deleted doc with missing id')
-      continue
-    }
-    const id = doc._id.toString()
-    // Skip doc if we already have an entry in entities
-    if (!entities.has(id)) {
-      const docIsInS3 = !!doc.inS3
-      let docLines
-
-      if (docIsInS3) {
-        const docPeek = await ProjectEntityHandler.promises.getDoc(
-          ObjectId(projectId),
-          doc._id,
-          { peek: true }
-        )
-        docLines = docPeek.lines
-      } else {
-        docLines = doc.lines
-      }
-
-      if (!docLines) {
-        throw new Error(`no doc lines for doc ${id} (inS3: ${docIsInS3})`)
-      }
-
-      // const ts = Number(
-      //   doc.deletedAt ? new Date(doc.deletedAt) : Date.now()
-      // )
-
-      if (doc.name && !SafePath.isCleanFilename(doc.name)) {
-        const newName = SafePath.clean(doc.name)
-        logger.warn(
-          { projectId, docId: id, origName: doc.name, newName },
-          'renaming invalid deleted file'
-        )
-        doc.name = newName
-      }
-
-      entities.set(id, {
-        // NOTE: adding the doc id to the file path to avoid collisions
-        path: `/_deleted/${id}/${doc.name}`,
-        name: doc.name || 'unnamed', // fallback for improperly deleted docs
-        deleted: true,
-        type: 'doc',
-        deletedAt: doc.deletedAt,
-        docLines: docLines.join('\n'),
-      })
-    }
-  }
-}
-
-/**
- * Read files deleted from a project, from the DeletedFile collection,
- * and add them to the entities map.
- *
- * These entities have a `deleted` property set to `true` and a `deletedAt` date.
- * The url is built later, from the project id and file id.
- *
- * @param {Map<string, Object>} entities
- * @param {string} projectId
- * @returns {Promise<void>}
- */
-async function readDeletedFiles(entities, projectId) {
-  const deletedFilesCursor = DeletedFile.find(
-    {
-      projectId: ObjectId(projectId),
-    },
-    // only read the fields we need to save memory
-    { _id: 1, name: 1, deletedAt: 1 }
-  )
-    .lean()
-    .cursor()
-
-  for await (const file of deletedFilesCursor) {
-    // skip malformed deleted file entries
-    if (!file?._id) {
-      logger.warn({ file }, 'skipping deleted file with missing id')
-      continue
-    }
-    const id = file._id.toString()
-    // TODO: check if it already exists?
-    if (!entities.has(id)) {
-      // const ts = Number(
-      //   file.deletedAt ? new Date(file.deletedAt) : Date.now()
-      // )
-
-      // TODO: would the hash be useful here?
-
-      if (file.name && !SafePath.isCleanFilename(file.name)) {
-        const newName = SafePath.clean(file.name)
-        logger.warn(
-          { projectId, fileId: id, origName: file.name, newName },
-          'renaming invalid deleted file'
-        )
-        file.name = newName
-      }
-
-      entities.set(id, {
-        // NOTE: adding the doc id to the file path to avoid collisions
-        path: `/_deleted/${id}/${file.name}`,
-        name: file.name,
-        deleted: true,
-        type: 'file',
-        deletedAt: file.deletedAt,
-      })
-    }
-  }
-}
-
-/**
- * Iterate through the sorted array of updates, pushing each one to Redis.
- *
- * In batches, tell project-history to pull the updates from Redis and process them,
- * so the process fails early if something can't be processed.
- *
- * @param {Array<AnyUpdate>} updates
- * @param {string} projectId
- * @param {string} projectHistoryId
- * @param {Map.<string, Object>} fileMap
- * @returns {Promise<void>}
- */
-async function sendUpdatesToProjectHistory(
-  updates,
-  projectId,
-  projectHistoryId,
-  fileMap
-) {
-  let multi = rclient.multi()
-  let counter = 0
-  let processed = 0
-  let size = 0
-
-  const projectHistoryKey =
-    settings.redis.project_history_migration.key_schema.projectHistoryOps({
-      projectId,
-    })
-
-  // clear out anything in the Redis queue for this project's history
-  multi.del(projectHistoryKey)
-
-  for (let update of updates) {
-    // read the content for each update stub from the archive
-    if (update.stub) {
-      update = await buildEditDocUpdate(projectHistoryId, update, fileMap)
-    }
-
-    // non-edit doc updates need string timestamps, not numbers
-    if (!('op' in update)) {
-      update.meta.ts = new Date(update.meta.ts).toISOString()
-    }
-
-    const updateJSON = JSON.stringify(update)
-    multi.rpush(projectHistoryKey, updateJSON)
-    counter++
-    processed++
-    size += updateJSON.length
-
-    // flush the history after every 1000 updates and start a new transaction
-    if (counter === 1000) {
-      logger.debug(
-        { processed, total: updates.length },
-        'sending updates to project history'
-      )
-      // execute the transaction
-      await util.promisify(multi.exec)()
-      // tell project-history to pull the updates from the Redis queue
-      await HistoryManager.promises.flushProject(projectId) // TODO: roll back if this fails?
-      counter = 0
-      size = 0
-      multi = rclient.multi()
-    } else if (size > 1024 * 1024) {
-      // queue entries in redis more frequently to reduce memory usage
-      await util.promisify(multi.exec)()
-      size = 0
-      multi = rclient.multi()
-    }
-  }
-
-  if (counter > 0) {
-    // execute the transaction
-    await util.promisify(multi.exec)()
-    // tell project-history to pull the updates from the Redis queue
-    await HistoryManager.promises.flushProject(projectId) // TODO: roll back if this fails?
-  }
-
-  // return the queue length so we can check that it is empty
-  const queueLength = await rclient.llen(projectHistoryKey)
-  return queueLength
-}
-
-/**
- * Compare two arrays of updates, with the earliest timestamp at the end first.
- *
- * @param {Array<AnyUpdate>} a
- * @param {Array<AnyUpdate>} b
- * @returns {number}
- */
-function earliestTimestampFirst(a, b) {
-  // both arrays are empty, leave them
-  if (!a.length && !b.length) {
-    return 0
-  }
-
-  // a is empty, move b before a
-  if (!a.length) {
-    return 1
-  }
-
-  // b is empty, don't move b before a
-  if (!b.length) {
-    return -1
-  }
-
-  const tsB = b[b.length - 1].meta.ts
-  const tsA = a[a.length - 1].meta.ts
-  // if the last item in b has a lower timestamp that the last item in a, move b above a
-  if (tsB < tsA) {
-    return 1
-  }
-  if (tsB > tsA) {
-    return -1
-  }
-  // use pathnames as secondary sort key, to make order deterministic for
-  // updates with the same timestamp
-  const pathnameB = b[b.length - 1].pathname
-  const pathnameA = a[a.length - 1].pathname
-  if (pathnameB < pathnameA) {
-    return 1
-  }
-  if (pathnameB > pathnameA) {
-    return -1
-  }
-  return 0 // shouldn't happen, because pathnames must be distinct
-}
-
-/**
- * Compare two updates, with the highest version number first
- *
- * @param {AnyUpdate} a
- * @param {AnyUpdate} b
- * @returns {number}
- */
-function decreasingDocVersion(a, b) {
-  if (b.v === a.v) {
-    throw new Error(`Matching version: ${b.v} ${a.v}`)
-    // return 0
-  }
-  // if b.v is greater than a.v, sort b above a
-  return b.v > a.v ? 1 : -1
-}
-
-/**
- * Create an array of queued updates for each doc/file, sorted by version
- *
- * @param {Array<AnyUpdate>} updates
- * @returns {Promise<Array<AnyUpdate>>}
- */
-async function sortUpdatesByQueue(updates) {
-  // build a queue of updates for each doc/file
-  const queues = {}
-
-  for (const update of updates) {
-    const docId = update.doc || update.file
-
-    if (!(docId in queues)) {
-      queues[docId] = []
-    }
-
-    queues[docId].push(update)
-  }
-
-  // convert the map to an array of queues
-  const values = Object.values(queues)
-
-  for (const queue of values) {
-    // sort each queue in place, with each update in decreasing version ofder
-    queue.sort(decreasingDocVersion)
-  }
-
-  return values
-}
-
-/**
- * Fetch all the content and updates for this project from track-changes, as a zip archive.
- *
- * @param {string} projectId
- * @param {string} tempFilePath
- * @returns
- */
-async function fetchTrackChangesArchive(projectId, tempFilePath) {
-  const writeStream = fs.createWriteStream(tempFilePath)
-
-  const url = `${settings.apis.trackchanges.url}/project/${projectId}/zip`
-
-  // exposed for debugging during full-project-history migration
-  const timeout =
-    parseInt(process.env.FETCH_TRACK_CHANGES_TIMEOUT, 10) || 2 * 60 * 1000
-
-  try {
-    await util.promisify(pipeline)(request(url, { timeout }), writeStream)
-  } catch (err) {
-    logger.error({ err }, 'Error fetching track changes archive')
-    throw err
-  }
-
-  const { size } = await fs.promises.stat(tempFilePath)
-  logger.info({ projectId, size }, 'fetched zip file from track-changes')
-}
-
-/**
- * Open the zip archive and build a Map of each entry in the archive, with the path as the key
- *
- * @param {string} filePath
- * @returns {Promise<Map<string, Object>>}
- */
-
-async function openTrackChangesArchive(filePath) {
-  const directory = await unzipper.Open.file(filePath)
-  return new Map(directory.files.map(file => [file.path, file]))
-}
-
-/**
- * Read the manifest data from the zip archive
- *
- * @param {Map<string, Object>} fileMap
- * @returns {Promise<Manifest>}
- */
-async function readTrackChangesManifest(fileMap) {
-  const manifestBuffer = await fileMap.get('manifest.json').buffer()
-
-  return JSON.parse(manifestBuffer.toString())
-}
-
-/**
- * Check that entities conform to the pathnames allowed by project history
- *
- * @param {Map<string, Object>} entities
- * @param {string} projectId
- */
-function validatePaths(entities, projectId) {
-  const pathErrors = []
-  for (const [id, entity] of entities) {
-    if (!SafePath.isCleanPath(entity.path)) {
-      pathErrors.push(
-        `${entity.type}:${id}${entity.deleted ? ' (deleted)' : ''} path:${
-          entity.path
-        }`
-      )
-    }
-  }
-  if (pathErrors.length) {
-    throw new OError('Invalid path in history migration', {
-      projectId,
-      pathErrors,
-    })
-  }
-}
-
-/**
- * Build an "add" update for an entity, with docLines or url set for the content.
- * This represents a doc or file being added to a project.
- *
- * @param {Object} entity
- * @param {string} entityId
- * @param {string} projectId
- * @param {string} projectHistoryId
- *
- * @returns {AddDocUpdate | AddFileUpdate}
- */
-function buildAddUpdate(entity, entityId, projectId, projectHistoryId) {
-  const ts = new ObjectId(entityId).getTimestamp()
-
-  const update = {
-    pathname: entity.path,
-    v: 0, // NOTE: only for sorting
-    meta: {
-      // source?
-      user_id: null, // TODO: assign the update to a system user?
-      ts: Number(ts),
-      origin: { kind: 'history-migration' },
-    },
-    projectHistoryId,
-  }
-
-  switch (entity.type) {
-    case 'doc': {
-      return {
-        doc: entityId,
-        ...update,
-        docLines: entity.docLines,
-      }
-    }
-
-    case 'file': {
-      // TODO: set a hash here?
-      return {
-        // type: 'external',
-        file: entityId,
-        ...update,
-        url: FilestoreHandler._buildUrl(projectId, entityId),
-      }
-    }
-
-    default:
-      throw new Error('Unknown entity type')
-  }
-}
-
-/**
- * Build a "delete" update for an entity, with new_pathname set to an empty string.
- * This represents a doc or file being deleted from a project.
- *
- * @param {Object} entity
- * @param {string} entityId
- * @param {string} projectId
- * @param {string} projectHistoryId
- * @returns DeleteUpdate
- */
-function buildDeleteUpdate(entity, entityId, projectId, projectHistoryId) {
-  const ts = entity.deletedAt || new Date()
-
-  const update = {
-    pathname: entity.path,
-    new_pathname: '', // empty path = deletion
-    v: Infinity, // NOTE: only for sorting
-    meta: {
-      user_id: null, // TODO: assign this to a system user?
-      ts: Number(ts),
-      origin: { kind: 'history-migration' },
-    },
-    projectHistoryId,
-  }
-
-  switch (entity.type) {
-    case 'doc':
-      return {
-        doc: entityId,
-        ...update,
-      }
-
-    case 'file':
-      return {
-        file: entityId,
-        ...update,
-      }
-
-    default:
-      throw new Error(`Unknown entity type ${entity.type}`)
-  }
-}
-
-/**
- * @typedef TrackedDocUpdateMeta
- * @property {string} user_id
- * @property {number} start_ts
- */
-
-/**
- * @typedef TrackedDocUpdate
- * @property {string} doc_id
- * @property {Array<Object>} op
- * @property {number} v
- * @property {TrackedDocUpdateMeta} meta
- */
-
-/**
- * Build an "edit" update, with op set to an array of operations from track-changes.
- *
- * This represents the contents of a doc being edited in a project.
- *
- * @param {string} projectHistoryId
- * @param {EditDocUpdateStub} updateStub
- * @param {Map.<string, Object>} fileMap
- *
- * @returns {Promise<EditDocUpdate>}
- */
-async function buildEditDocUpdate(projectHistoryId, updateStub, fileMap) {
-  const buffer = await fileMap.get(updateStub.path).buffer()
-
-  /**
-   * @type TrackedDocUpdate
-   */
-  const data = JSON.parse(buffer.toString())
-  let userId = data.meta.user_id
-  if (userId === 'anonymous-user' || userId === 'null') {
-    userId = null
-  }
-  if (userId != null && !/^[0-9a-f]{24}$/.test(userId)) {
-    throw new OError('Bad user id in ShareLaTeX history edit update', {
-      userId,
-    })
-  }
-
-  return {
-    doc: data.doc_id,
-    op: data.op, // NOTE: this is an array of operations
-    v: data.v,
-    lastV: data.v - 1,
-    meta: {
-      user_id: userId,
-      ts: data.meta.start_ts, // TODO: use data.meta.end_ts or update.ts?
-      pathname: updateStub.pathname,
-      doc_length: updateStub.doc_length,
-      origin: { kind: 'history-migration' },
-    },
-    projectHistoryId,
-  }
-}
-
-/**
- * Build a stub for an "edit" update, with all the metadata but not the actual operations.
- *
- * This represents a doc being edited in a project, with enough information for sorting,
- * but avoids loading the actual operations from the zip archive until they're needed,
- * so as not to run out of memory if the project's history is large.
- *
- * @param {ManifestUpdate} update
- * @param {Entity} entity
- * @param {string} docId
- * @returns {EditDocUpdateStub}
- */
-function buildEditUpdateStub(update, entity, docId) {
-  return {
-    stub: true,
-    doc: docId,
-    v: update.version,
-    path: update.path,
-    pathname: entity.path,
-    doc_length: update.doc_length,
-    meta: {
-      ts: update.ts,
-      origin: { kind: 'history-migration' },
-    },
-  }
-}
-
-/**
- * Build the sorted array of updates to be sent to project-history.
- *
- * 1. Process all the added and edited files from the track-changes archive.
- * 2. Process the other files from the project that have been added, and maybe deleted, without any edits.
- *
- * @param {string} projectId
- * @param {string} projectHistoryId
- * @param {Manifest} manifest
- * @param {Map.<string, Entity>} entities
- * @param {Map.<string, Object>} fileMap
- * @returns {Promise<Array<AnyUpdate>>}
- */
-async function buildUpdates(
-  projectId,
-  projectHistoryId,
-  manifest,
-  entities,
-  fileMap
-) {
-  /**
-   * @type Array<AnyUpdate>
-   */
-  const updates = []
-
-  // keep a list of doc ids which have updates in track-changes
-  const updatedDocs = new Set()
-
-  // process the existing docs with updates, from track-changes
-  for (const doc of manifest.docs) {
-    const entity = entities.get(doc.id)
-
-    if (!entity) {
-      throw new Error(`Entity not found for ${doc.id}`)
-    }
-
-    if (!entity.path) {
-      throw new Error(`Path not found for ${doc.id}`)
-    }
-
-    // add the initial content
-    const contentStart = doc.content.start
-
-    const buffer = await fileMap.get(contentStart.path).buffer()
-
-    /**
-     * @type AddDocUpdate
-     */
-    const update = {
-      doc: doc.id,
-      pathname: entity.path,
-      v: contentStart.version - 1,
-      meta: {
-        user_id: null, // TODO: assign this to a system user?
-        ts: Number(ObjectId(doc.id).getTimestamp()),
-        origin: { kind: 'history-migration' },
-      },
-      projectHistoryId,
-      docLines: buffer.toString(),
-    }
-
-    updates.push(update)
-
-    // push the update onto the array of updates
-    for (const update of doc.updates) {
-      updates.push(buildEditUpdateStub(update, entity, doc.id))
-    }
-
-    updatedDocs.add(doc.id)
-  }
-
-  // process the docs which have been added/deleted without any updates being recorded
-  for (const [id, entity] of entities.entries()) {
-    if (entity.deleted) {
-      // deleted entity
-
-      // add the doc/file
-      if (!updatedDocs.has(id)) {
-        updates.push(buildAddUpdate(entity, id, projectId, projectHistoryId))
-      }
-
-      // delete the doc/file again (there may be updates added between adding and deleting)
-      updates.push(buildDeleteUpdate(entity, id, projectId, projectHistoryId))
-    } else {
-      if (!updatedDocs.has(id)) {
-        // add "not deleted" doc that isn't in the manifest either
-        updates.push(buildAddUpdate(entity, id, projectId, projectHistoryId))
-      }
-    }
-  }
-
-  return updates
-}
-
-/**
- * Remove the `overleaf.history` object from the project and tell project-history to delete everything for this project.
- * (note: project-history may not delete the actual history data yet, but it will at least delete the cached history id)
- *
- * @param {string} projectId
- * @returns {Promise<void>}
- */
-async function deleteProjectHistory(projectId) {
-  // look up the history id from the project
-  const historyId = await ProjectHistoryHandler.promises.getHistoryId(projectId)
-  // delete the history from project-history and history-v1
-  await HistoryManager.promises.deleteProject(projectId, historyId)
-  // TODO: send a message to document-updater?
-  await ProjectHistoryHandler.promises.unsetHistory(projectId)
-}
-
-/**
- * Send the updates from the track changes zip file to project history
- *
- * @param {string} projectId
- * @param {string} projectHistoryId
- * @param {Array<AnyUpdate>} updates
- * @param {Map.<string, Object>} fileMap
- */
-async function migrateTrackChangesUpdates(
-  projectId,
-  projectHistoryId,
-  updates,
-  fileMap
-) {
-  // Build a queue for each doc, sorted by version (and by timestamp within each version)
-  const queues = await sortUpdatesByQueue(updates)
-
-  const sortedUpdates = []
-
-  let item
-  do {
-    // Find the earliest item from the tail of all queues
-    queues.sort(earliestTimestampFirst)
-    item = queues[0].pop()
-    if (item) {
-      sortedUpdates.push(item)
-    }
-  } while (item)
-
-  // NOTE: leaving the version string code commented out, in case it ends up being needed
-  // let majorVersion = 0
-  // let minorVersion = 0
-  for (const update of sortedUpdates) {
-    // increment majorVersion if this is a file change
-    if (!('op' in update)) {
-      // remove v (only used for sorting)
-      delete update.v
-
-      // set version
-      // majorVersion++
-      // // minorVersion = 0
-      // update.version = `${majorVersion}.${minorVersion}` // NOTE: not set as project-history doesn't need it and could cause problems if it gets higher than project.version
-    }
-    // increment minorVersion after every update
-    // minorVersion++
-  }
-
-  // add each update to the Redis queue for project-history to process
-  logger.debug(
-    { projectId, projectHistoryId },
-    'Sending updates for project to Redis'
-  )
-
-  const remainingQueueLength = await sendUpdatesToProjectHistory(
-    sortedUpdates,
-    projectId,
-    projectHistoryId,
-    fileMap
-  )
-  // Failure will cause queued updates to be deleted (in the catch below)
-
-  logger.debug(
-    {
-      projectId,
-      projectHistoryId,
-      remainingQueueLength,
-    },
-    'Updates sent to project-history'
-  )
-
-  if (remainingQueueLength > 0) {
-    throw new Error('flush to project-history did not complete')
-  }
-
-  // TODO: roll back if any of the following fail?
-
-  // TODO: check that the Redis queue is empty?
-
-  // Clear any old entries in the main project history queue (these will not
-  // have a history id)
-  await HistoryManager.promises.flushProject(projectId)
-}
-
-/**
- * Add the zip file from track changes to the project file tree.
- * We may be able to recover a failed history from the zip file in future.
- *
- * @param {string} projectId
- * @param {string} rootFolderId
- * @param {string} tempFilePath
- */
-
-async function uploadTrackChangesArchiveToProject(
-  projectId,
-  rootFolderId,
-  tempFilePath
-) {
-  const { size } = await fs.promises.stat(tempFilePath)
-  if (size > settings.maxUploadSize) {
-    throw new FileTooLargeError({
-      message: 'track-changes archive exceeds maximum size for archiving',
-      info: { size },
-    })
-  }
-  const { fileRef } = await ProjectEntityUpdateHandler.promises.addFile(
-    projectId,
-    rootFolderId, // project.rootFolder[0]._id,
-    `OverleafHistory-${new Date().toISOString().substring(0, 10)}.zip`,
-    tempFilePath,
-    null,
-    null, // no owner
-    null // no source
-  )
-  logger.debug(
-    { projectId, fileRef },
-    'Uploaded track-changes zip archive to project due to error in migration'
-  )
-}
-
-/**
- * Check all updates for invalid characters (nonBMP or null) and substitute
- * the unicode replacement character if options.fixInvalidCharacters is true,
- * otherwise throw an exception.
- * @param {Array<AnyUpdate>} updates
- * @param {string} projectId
- * @param {Object} options
- */
-function validateUpdates(updates, projectId, options) {
-  const replace = options.fixInvalidCharacters
-  // check for invalid characters
-  function containsBadChars(str) {
-    return /[\uD800-\uDBFF]/.test(str) || str.indexOf('\x00') !== -1
-  }
-  // Replace invalid characters so that they will be accepted by history_v1.
-  function sanitise(str) {
-    if (replace) {
-      return str.replace(/[\uD800-\uDFFF]/g, '\uFFFD').replace('\x00', '\uFFFD')
-    } else {
-      throw new Error('invalid character in content')
-    }
-  }
-  // Check size of doclines in update against max size allowed by history_v1.
-  // This catches docs which are too large when created, but not when they
-  // go over the limit due to edits.
-  function checkSize(update) {
-    if (update?.docLines?.length > settings.max_doc_length) {
-      throw new FileTooLargeError({
-        message: 'docLines exceeds maximum size for history',
-        info: { docId: update.doc, size: update.docLines.length },
-      })
-    }
-  }
-  let latestTimestamp = 0
-  // Iterate over the all the updates and their doclines or ops
-  for (const update of updates) {
-    checkSize(update)
-    // Find the timestamp of the most recent edit (either adding a doc or editing a doc)
-    // we exclude deletions as these are created in the migration and we didn't record
-    // the deletion time for older files.
-    const isDeleteUpdate = update.new_pathname === ''
-    if (
-      update.doc &&
-      !isDeleteUpdate &&
-      update.meta.ts &&
-      update.meta.ts > latestTimestamp
-    ) {
-      latestTimestamp = update.meta.ts
-    }
-    if (update.docLines && containsBadChars(update.docLines)) {
-      logger.debug({ update, replace }, 'invalid character in docLines')
-      update.docLines = sanitise(update.docLines)
-    }
-    if (update.op) {
-      for (const op of update.op) {
-        if (op.i && containsBadChars(op.i)) {
-          logger.debug({ update, replace }, 'invalid character in insert op')
-          op.i = sanitise(op.i)
-        }
-        if (op.d && containsBadChars(op.d)) {
-          logger.debug({ update, replace }, 'invalid character in delete op')
-          op.d = sanitise(op.d)
-        }
-      }
-    }
-  }
-  logger.debug(
-    { projectId, latestTimestamp, date: new Date(latestTimestamp) },
-    'timestamp of most recent edit'
-  )
-  if (options.cutoffDate && new Date(latestTimestamp) > options.cutoffDate) {
-    throw new Error('project was edited after cutoff date')
-  }
-}
-
-/**
- * Migrate a project's history from track-changes to project-history
- *
- * @param {string} projectId
- *
- * @returns {Promise<void>}
- */
-async function migrateProjectHistory(projectId, options = {}) {
-  await fse.ensureDir(settings.path.projectHistories)
-  const projectHistoriesDir = await fs.promises.realpath(
-    settings.path.projectHistories
-  )
-  const tempDir = await fs.promises.mkdtemp(projectHistoriesDir + path.sep)
-  const tempFilePath = path.join(tempDir, 'project.zip')
-
-  try {
-    // fetch the zip archive of rewound content and updates from track-changes
-    // store the zip archive to disk, open it and build a Map of the entries
-    if (options.importZipFilePath) {
-      // use an existing track-changes archive on disk
-      logger.debug(
-        { src: options.importZipFilePath, dst: tempFilePath },
-        'importing zip file'
-      )
-      await fs.promises.copyFile(options.importZipFilePath, tempFilePath)
-      const { size } = await fs.promises.stat(tempFilePath)
-      logger.info({ projectId, size }, 'imported zip file from disk')
-    } else {
-      await fetchTrackChangesArchive(projectId, tempFilePath)
-    }
-    const fileMap = await openTrackChangesArchive(tempFilePath)
-
-    // read the manifest from the zip archive
-    const manifest = await readTrackChangesManifest(fileMap)
-
-    // check that the project id in the manifest matches
-    // to be sure we are using the correct zip file
-    if (manifest.projectId !== projectId) {
-      throw new Error(`Incorrect projectId: ${manifest.projectId}`)
-    }
-
-    // load the Project from MongoDB
-    const project = await ProjectGetter.promises.getProject(projectId)
-
-    // create a history id for this project
-    const oldProjectHistoryId = _.get(project, 'overleaf.history.id')
-
-    // throw an error if there is already a history associated with the project
-    if (oldProjectHistoryId) {
-      throw new Error(
-        `Project ${projectId} already has history ${oldProjectHistoryId}`
-      )
-    }
-
-    try {
-      // initialize a new project history and use the history id
-      // NOTE: not setting the history id on the project yet
-      const projectHistoryId = await HistoryManager.promises.initializeProject(
-        projectId
-      )
-
-      try {
-        // build a Map of the entities (docs and fileRefs) currently in the project,
-        // with _id as the key
-        const entities = await processRootFolder(project)
-
-        // find all the deleted docs for this project and add them to the entity map
-        await readDeletedDocs(entities, projectId)
-
-        // find all the deleted files for this project and add them to the entity map
-        await readDeletedFiles(entities, projectId)
-
-        // check that the paths will not be rejected
-        validatePaths(entities, projectId)
-
-        // build the array of updates that make up the new history for this project
-        const updates = await buildUpdates(
-          projectId,
-          projectHistoryId,
-          manifest,
-          entities,
-          fileMap
-        )
-
-        // check that the updates don't contain any characters that will be rejected by history_v1.
-        validateUpdates(updates, projectId, options)
-
-        if (updates.length) {
-          await migrateTrackChangesUpdates(
-            projectId,
-            projectHistoryId,
-            updates,
-            fileMap
-          )
-        }
-      } catch (error) {
-        if (options?.archiveOnFailure) {
-          // on error, optionally store the zip file in the project for future reference
-          logger.debug(
-            { projectId, error },
-            'Error sending track-changes updates to project history, attempting to archive zip file in project'
-          )
-          try {
-            await uploadTrackChangesArchiveToProject(
-              projectId,
-              project.rootFolder[0]._id,
-              tempFilePath
-            )
-          } catch (error) {
-            if (error instanceof InvalidNameError) {
-              logger.info({ projectId }, 'zip file already archived in project')
-            } else {
-              throw error
-            }
-          } finally {
-            // roll back the last updated timestamp and user
-            logger.debug(
-              { projectId },
-              'rolling back last updated time after uploading zip file'
-            )
-            await ProjectUpdateHandler.promises.resetUpdated(
-              projectId,
-              project.lastUpdated,
-              project.lastUpdatedBy
-            )
-          }
-          // set the overleaf.history.zipFileArchivedInProject flag for future reference
-          await ProjectHistoryHandler.promises.setMigrationArchiveFlag(
-            projectId
-          )
-          // we consider archiving the zip file as "success" (at least we've given up on attempting
-          // to migrate the history) so we don't rethrow the error and continue to initialise the new
-          // empty history below.
-        } else {
-          // if we're not archiving the zip file then we rethrown the error to fail the migration
-          throw error
-        }
-      }
-
-      // set the project's history id once the updates have been successfully processed
-      // (or we have given up and archived the zip file in the project).
-      logger.debug(
-        { projectId, projectHistoryId },
-        'Setting history id on project'
-      )
-      await ProjectHistoryHandler.promises.setHistoryId(
-        projectId,
-        projectHistoryId
-      )
-
-      try {
-        // tell document updater to reload docs with the new history id
-        logger.debug({ projectId }, 'Asking document-updater to clear project')
-        await DocumentUpdaterHandler.promises.flushProjectToMongoAndDelete(
-          projectId
-        )
-
-        // run a project history resync in case any changes have arrived since the migration
-        logger.debug(
-          { projectId },
-          'Asking project-history to force resync project'
-        )
-
-        await HistoryManager.promises.resyncProject(projectId, {
-          force: true,
-          origin: { kind: 'history-migration' },
-        })
-      } catch (error) {
-        if (options.forceNewHistoryOnFailure) {
-          logger.warn(
-            { projectId },
-            'failed to resync project, forcing new history'
-          )
-        } else {
-          throw error
-        }
-      }
-      logger.debug(
-        { projectId },
-        'Switching on full project history display for project'
-      )
-      // Set the display to v2 history but allow downgrading (second argument allowDowngrade = true)
-      await ProjectHistoryHandler.promises.upgradeHistory(projectId, true)
-    } catch (error) {
-      // delete the history id again if something failed?
-      logger.warn(
-        OError.tag(
-          error,
-          'Something went wrong flushing and resyncing project; clearing full project history for project',
-          { projectId }
-        )
-      )
-      await deleteProjectHistory(projectId)
-
-      throw error
-    }
-  } finally {
-    // clean up the temporary directory
-    await fse.remove(tempDir)
-  }
-}

services/web/modules/history-migration/index.js

@@ -1 +0,0 @@
-module.exports = {}

services/web/modules/history-migration/test/unit/src/ProjectHistoryControllerTests.js

@@ -1,346 +0,0 @@
-const sinon = require('sinon')
-const nock = require('nock')
-const { expect } = require('chai')
-const fs = require('fs')
-const path = require('path')
-const SandboxedModule = require('sandboxed-module')
-const { ObjectId } = require('mongodb')
-const unzipper = require('unzipper')
-
-const modulePath = '../../../app/src/ProjectHistoryController'
-
-describe('ProjectHistoryController', function () {
-  const projectId = ObjectId('611bd20c5d76a3c1bd0c7c13')
-  const deletedFileId = ObjectId('60f6e92c6c14d84fb7a71ae1')
-  const historyId = 123
-
-  let clock
-  const now = new Date(Date.UTC(2021, 1, 1, 0, 0)).getTime()
-
-  before(async function () {
-    clock = sinon.useFakeTimers({
-      now,
-      shouldAdvanceTime: true,
-    })
-  })
-
-  after(function () {
-    // clock.runAll()
-    clock.restore()
-  })
-
-  beforeEach(function () {
-    this.db = {
-      users: {
-        countDocuments: sinon.stub().yields(),
-      },
-    }
-
-    this.project = {
-      _id: ObjectId('611bd20c5d76a3c1bd0c7c13'),
-      name: 'My Test Project',
-      rootDoc_id: ObjectId('611bd20c5d76a3c1bd0c7c15'),
-      rootFolder: [
-        {
-          _id: ObjectId('611bd20c5d76a3c1bd0c7c12'),
-          name: 'rootFolder',
-          folders: [
-            {
-              _id: ObjectId('611bd242e64281c13303d6b5'),
-              name: 'a folder',
-              folders: [
-                {
-                  _id: ObjectId('611bd247e64281c13303d6b7'),
-                  name: 'a subfolder',
-                  folders: [],
-                  fileRefs: [],
-                  docs: [
-                    {
-                      _id: ObjectId('611bd24ee64281c13303d6b9'),
-                      name: 'a renamed file in a subfolder.tex',
-                    },
-                  ],
-                },
-              ],
-              fileRefs: [],
-              docs: [],
-            },
-            {
-              _id: ObjectId('611bd34ee64281c13303d6be'),
-              name: 'images',
-              folders: [],
-              fileRefs: [
-                {
-                  _id: ObjectId('611bd2bce64281c13303d6bb'),
-                  name: 'overleaf-white.svg',
-                  linkedFileData: {
-                    provider: 'url',
-                    url: 'https://cdn.overleaf.com/img/ol-brand/overleaf-white.svg',
-                  },
-                  created: '2021-08-17T15:16:12.753Z',
-                },
-              ],
-              docs: [],
-            },
-          ],
-          fileRefs: [
-            {
-              _id: ObjectId('611bd20c5d76a3c1bd0c7c19'),
-              name: 'universe.jpg',
-              linkedFileData: null,
-              created: '2021-08-17T15:13:16.400Z',
-            },
-          ],
-          docs: [
-            {
-              _id: ObjectId('611bd20c5d76a3c1bd0c7c15'),
-              name: 'main.tex',
-            },
-            {
-              _id: ObjectId('611bd20c5d76a3c1bd0c7c17'),
-              name: 'references.bib',
-            },
-          ],
-        },
-      ],
-      compiler: 'pdflatex',
-      description: '',
-      deletedDocs: [],
-      members: [],
-      invites: [],
-      owner: {
-        _id: ObjectId('611572e24bff88527f61dccd'),
-        first_name: 'Test',
-        last_name: 'User',
-        email: 'test@example.com',
-        privileges: 'owner',
-        signUpDate: '2021-08-12T19:13:38.462Z',
-      },
-      features: {},
-    }
-
-    this.multi = {
-      del: sinon.stub(),
-      rpush: sinon.stub(),
-      exec: sinon.stub().yields(null, 1),
-    }
-
-    const { docs, folders } = this.project.rootFolder[0]
-
-    const allDocs = [...docs]
-
-    const processFolders = folders => {
-      for (const folder of folders) {
-        for (const doc of folder.docs) {
-          allDocs.push(doc)
-        }
-
-        if (folder.folders) {
-          processFolders(folder.folders)
-        }
-      }
-    }
-
-    processFolders(folders)
-
-    allDocs.forEach(doc => {
-      doc.lines = [`this is the contents of ${doc.name}`]
-    })
-
-    // handle Doc.find().lean().cursor()
-    this.findDocs = sinon.stub().returns({
-      lean: sinon.stub().returns({
-        cursor: sinon.stub().returns(allDocs),
-      }),
-    })
-
-    // handle await Doc.findOne().lean() - single result, no cursor required
-    this.findOneDoc = sinon.stub().callsFake(id => {
-      const result = allDocs.find(doc => {
-        return doc._id.toString() === id.toString()
-      })
-      return { lean: sinon.stub().resolves(result) }
-    })
-
-    this.deletedFiles = [
-      {
-        _id: deletedFileId,
-        name: 'testing.tex',
-        deletedAt: new Date(),
-      },
-    ]
-
-    // handle DeletedFile.find().lean().cursor()
-    this.findDeletedFiles = sinon.stub().returns({
-      lean: sinon
-        .stub()
-        .returns({ cursor: sinon.stub().returns(this.deletedFiles) }),
-    })
-
-    this.ProjectGetter = {
-      promises: {
-        getProject: sinon.stub().resolves(this.project),
-      },
-    }
-
-    this.FileStoreHandler = {
-      _buildUrl: (projectId, fileId) =>
-        `http://filestore.test/${projectId}/${fileId}`,
-    }
-
-    this.ProjectHistoryHandler = {
-      promises: {
-        setHistoryId: sinon.stub(),
-        upgradeHistory: sinon.stub(),
-      },
-    }
-
-    this.ProjectEntityUpdateHandler = {
-      promises: {
-        resyncProjectHistory: sinon.stub(),
-      },
-    }
-
-    this.DocumentUpdaterHandler = {
-      promises: {
-        flushProjectToMongoAndDelete: sinon.stub(),
-      },
-    }
-
-    this.HistoryManager = {
-      promises: {
-        resyncProject: sinon.stub(),
-        flushProject: sinon.stub(),
-        initializeProject: sinon.stub().resolves(historyId),
-      },
-    }
-
-    this.settings = {
-      redis: {
-        project_history_migration: {
-          key_schema: {
-            projectHistoryOps({ projectId }) {
-              return `ProjectHistory:Ops:{${projectId}}` // NOTE: the extra braces are intentional
-            },
-          },
-        },
-      },
-      apis: {
-        documentupdater: {
-          url: 'http://document-updater',
-        },
-        trackchanges: {
-          url: 'http://track-changes',
-        },
-        project_history: {
-          url: 'http://project-history',
-        },
-      },
-      path: {
-        projectHistories: 'data/projectHistories',
-      },
-    }
-
-    this.ProjectHistoryController = SandboxedModule.require(modulePath, {
-      requires: {
-        '../../../../app/src/Features/Project/ProjectGetter':
-          this.ProjectGetter,
-        '../../../../app/src/Features/FileStore/FileStoreHandler':
-          this.FileStoreHandler,
-        '../../../../app/src/Features/Project/ProjectHistoryHandler':
-          this.ProjectHistoryHandler,
-        '../../../../app/src/Features/Project/ProjectUpdateHandler':
-          this.ProjectUpdateHandler,
-        '../../../../app/src/Features/Project/ProjectEntityUpdateHandler':
-          this.ProjectEntityUpdateHandler,
-        '../../../../app/src/Features/History/HistoryManager':
-          this.HistoryManager,
-        '../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler':
-          this.DocumentUpdaterHandler,
-        '../../../../app/src/models/Doc': {
-          Doc: {
-            find: this.findDocs,
-            findOne: this.findOneDoc,
-          },
-        },
-        '../../../../app/src/models/DeletedFile': {
-          DeletedFile: {
-            find: this.findDeletedFiles,
-          },
-        },
-        '../../../../app/src/infrastructure/mongodb': {
-          db: this.db,
-        },
-        '../../../../app/src/infrastructure/Mongoose': {
-          Schema: {
-            ObjectId: sinon.stub(),
-            Types: {
-              Mixed: sinon.stub(),
-            },
-          },
-        },
-        '../../../../app/src/infrastructure/RedisWrapper': {
-          client: () => ({
-            multi: () => this.multi,
-            llen: sinon.stub().resolves(0),
-          }),
-        },
-        unzipper: {
-          Open: {
-            file: () =>
-              unzipper.Open.file(
-                path.join(__dirname, 'data/track-changes-project.zip')
-              ),
-          },
-        },
-        '@overleaf/settings': this.settings,
-      },
-    })
-  })
-
-  afterEach(function () {
-    nock.cleanAll()
-  })
-
-  it('migrates a project history', async function () {
-    const readStream = fs.createReadStream(
-      path.join(__dirname, 'data/track-changes-project.zip')
-    )
-
-    nock(this.settings.apis.trackchanges.url)
-      .get(`/project/${projectId}/zip`)
-      .reply(200, readStream)
-
-    nock(this.settings.apis.project_history.url)
-      .post(`/project`)
-      .reply(200, { project: { id: historyId } })
-
-    await this.ProjectHistoryController.migrateProjectHistory(
-      projectId.toString(),
-      5
-    )
-
-    expect(this.multi.exec).to.have.been.calledOnce
-    expect(this.ProjectHistoryHandler.promises.setHistoryId).to.have.been
-      .calledOnce
-    // expect(this.ProjectEntityUpdateHandler.promises.resyncProjectHistory).to
-    //   .have.been.calledOnce
-    expect(this.HistoryManager.promises.flushProject).to.have.been.calledTwice
-    expect(this.multi.rpush).to.have.callCount(12)
-
-    const args = this.multi.rpush.args
-
-    const snapshotPath = path.join(
-      __dirname,
-      'data/migrate-project-history.snapshot.json'
-    )
-
-    // const snapshot = JSON.stringify(args, null, 2)
-    // await fs.promises.writeFile(snapshotPath, snapshot)
-
-    const json = await fs.promises.readFile(snapshotPath, 'utf-8')
-    const expected = JSON.parse(json)
-
-    expect(args).to.deep.equal(expected)
-  })
-})

services/web/modules/history-migration/test/unit/src/data/migrate-project-history.snapshot.json

@@ -1,50 +0,0 @@
-[
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"file\":\"60f6e92c6c14d84fb7a71ae1\",\"pathname\":\"/_deleted/60f6e92c6c14d84fb7a71ae1/testing.tex\",\"meta\":{\"user_id\":null,\"ts\":\"2021-07-20T15:18:04.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"url\":\"http://filestore.test/611bd20c5d76a3c1bd0c7c13/60f6e92c6c14d84fb7a71ae1\"}"
-  ],
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"file\":\"60f6e92c6c14d84fb7a71ae1\",\"pathname\":\"/_deleted/60f6e92c6c14d84fb7a71ae1/testing.tex\",\"new_pathname\":\"\",\"meta\":{\"user_id\":null,\"ts\":\"2021-02-01T00:00:00.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}"
-  ],
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"doc\":\"611bd20c5d76a3c1bd0c7c15\",\"pathname\":\"/main.tex\",\"meta\":{\"user_id\":null,\"ts\":\"2021-08-17T15:13:16.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"docLines\":\"\\\\documentclass{article}\\n\\\\usepackage[utf8]{inputenc}\\n\\n\\\\title{My Test Project}\\n\\\\author{alf.eaton+dev }\\n\\\\date{7 2021}\\n\\n\\\\usepackage{natbib}\\n\\\\usepackage{graphicx}\\n\\n\\\\begin{document}\\n\\n\\\\maketitle\\n\\n\\\\section{Introduction}\\nThere is a theory which states that if ever anyone discovers exactly what the Universe is for and why it is here, it will instantly disappear and be replaced by something even more bizarre and inexplicable.\\nThere is another theory which states that this has already happened.\\n\\n\\\\begin{figure}[h!]\\n\\\\centering\\n\\\\includegraphics[scale=1.7]{universe}\\n\\\\caption{The Universe}\\n\\\\label{fig:universe}\\n\\\\end{figure}\\n\\n\\\\section{Conclusion}\\n``I always thought something was fundamentally wrong with the universe'' \\\\citep{adams1995hitchhiker}\\n\\n\\\\bibliographystyle{plain}\\n\\\\bibliography{references}\\n\\\\end{document}\\n\"}"
-  ],
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"doc\":\"611bd20c5d76a3c1bd0c7c17\",\"pathname\":\"/references.bib\",\"meta\":{\"user_id\":null,\"ts\":\"2021-08-17T15:13:16.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"docLines\":\"this is the contents of references.bib\"}"
-  ],
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"file\":\"611bd20c5d76a3c1bd0c7c19\",\"pathname\":\"/universe.jpg\",\"meta\":{\"user_id\":null,\"ts\":\"2021-08-17T15:13:16.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"url\":\"http://filestore.test/611bd20c5d76a3c1bd0c7c13/611bd20c5d76a3c1bd0c7c19\"}"
-  ],
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"doc\":\"611bd20c5d76a3c1bd0c7c15\",\"op\":[{\"p\":487,\"i\":\"\\n\\nAdding some text here.\"}],\"v\":1,\"lastV\":0,\"meta\":{\"user_id\":\"611572e24bff88527f61dccd\",\"ts\":1629213228148,\"pathname\":\"/main.tex\",\"doc_length\":805,\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}"
-  ],
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"doc\":\"611bd20c5d76a3c1bd0c7c15\",\"op\":[{\"p\":678,\"d\":\" something\"}],\"v\":2,\"lastV\":1,\"meta\":{\"user_id\":\"611572e24bff88527f61dccd\",\"ts\":1629213235181,\"pathname\":\"/main.tex\",\"doc_length\":829,\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}"
-  ],
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"doc\":\"611bd20c5d76a3c1bd0c7c15\",\"op\":[{\"d\":\" \",\"p\":722},{\"i\":\"\\n\",\"p\":722}],\"v\":3,\"lastV\":2,\"meta\":{\"user_id\":\"611572e24bff88527f61dccd\",\"ts\":1629213239472,\"pathname\":\"/main.tex\",\"doc_length\":819,\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}"
-  ],
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"doc\":\"611bd20c5d76a3c1bd0c7c15\",\"op\":[{\"p\":750,\"i\":\"\\n\\nAdding some text after deleting some text.\"}],\"v\":7,\"lastV\":6,\"meta\":{\"user_id\":\"611572e24bff88527f61dccd\",\"ts\":1629213241498,\"pathname\":\"/main.tex\",\"doc_length\":819,\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}"
-  ],
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"doc\":\"611bd24ee64281c13303d6b9\",\"pathname\":\"/a folder/a subfolder/a renamed file in a subfolder.tex\",\"meta\":{\"user_id\":null,\"ts\":\"2021-08-17T15:14:22.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"docLines\":\"\"}"
-  ],
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"doc\":\"611bd24ee64281c13303d6b9\",\"op\":[{\"p\":0,\"i\":\"Adding some content to the file in the subfolder.\"}],\"v\":2,\"lastV\":1,\"meta\":{\"user_id\":\"611572e24bff88527f61dccd\",\"ts\":1629213266076,\"pathname\":\"/a folder/a subfolder/a renamed file in a subfolder.tex\",\"doc_length\":0,\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123}"
-  ],
-  [
-    "ProjectHistory:Ops:{611bd20c5d76a3c1bd0c7c13}",
-    "{\"file\":\"611bd2bce64281c13303d6bb\",\"pathname\":\"/images/overleaf-white.svg\",\"meta\":{\"user_id\":null,\"ts\":\"2021-08-17T15:16:12.000Z\",\"origin\":{\"kind\":\"history-migration\"}},\"projectHistoryId\":123,\"url\":\"http://filestore.test/611bd20c5d76a3c1bd0c7c13/611bd2bce64281c13303d6bb\"}"
-  ]
-]

services/web/package.json

@@ -243,7 +243,6 @@
     "scroll-into-view-if-needed": "^2.2.25",
     "tsscmp": "^1.0.6",
     "underscore": "^1.13.1",
-    "unzipper": "^0.10.11",
     "utf-8-validate": "^5.0.2",
     "uuid": "^3.0.1",
     "valid-data-url": "^2.0.0",

services/web/scripts/fix_oversized_docs.js

@@ -147,7 +147,6 @@ async function deleteDocFromRedis(projectId, docId) {
     `UnflushedTime:{${docId}}`,
     `Pathname:{${docId}}`,
     `ProjectHistoryId:{${docId}}`,
-    `ProjectHistoryType:{${docId}}`,
     `PendingUpdates:{${docId}}`,
     `lastUpdatedAt:{${docId}}`,
     `lastUpdatedBy:{${docId}}`

services/web/scripts/history/clean_sl_history_data.js

@@ -1,60 +0,0 @@
-const { waitForDb, db } = require('../../app/src/infrastructure/mongodb')
-
-async function main() {
-  await checkAllProjectsAreMigrated()
-  await setAllowDowngradeToFalse()
-  await deleteHistoryCollections()
-  console.log('Legacy history data cleaned up successfully')
-  process.exit(0)
-}
-
-async function checkAllProjectsAreMigrated() {
-  console.log('checking all projects are migrated to Full Project History')
-
-  const count = await db.projects.countDocuments({
-    'overleaf.history.display': { $ne: true },
-  })
-
-  if (count === 0) {
-    console.log('All projects are migrated to Full Project History')
-  } else {
-    console.error(
-      `There are ${count} projects that are not migrated to Full Project History` +
-        ` please complete the migration before running this script again.`
-    )
-    process.exit(1)
-  }
-}
-
-async function setAllowDowngradeToFalse() {
-  console.log('unsetting `allowDowngrade` flag in all projects')
-  await db.projects.updateMany(
-    {
-      'overleaf.history.id': { $exists: true },
-      'overleaf.history.allowDowngrade': true,
-    },
-    { $unset: { 'overleaf.history.allowDowngrade': 1 } }
-  )
-  console.log('unsetting `allowDowngrade` flag in all projects - Done')
-}
-
-async function deleteHistoryCollections() {
-  console.log('removing `docHistory` data')
-  await db.docHistory.deleteMany({})
-  console.log('removing `docHistory` data - Done')
-
-  console.log('removing `docHistoryIndex` data')
-  await db.docHistoryIndex.deleteMany({})
-  console.log('removing `docHistoryIndex` data - Done')
-
-  console.log('removing `projectHistoryMetaData` data')
-  await db.projectHistoryMetaData.deleteMany({})
-  console.log('removing `projectHistoryMetaData` data - Done')
-}
-
-waitForDb()
-  .then(main)
-  .catch(err => {
-    console.error(err)
-    process.exit(1)
-  })

services/web/scripts/history/count_project_history_categories.js

@@ -1,84 +0,0 @@
-const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
-const VERBOSE_PROJECT_NAMES = process.env.VERBOSE_PROJECT_NAMES === 'true'
-const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 50
-const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 500
-const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
-// persist fallback in order to keep batchedUpdate in-sync
-process.env.BATCH_SIZE = BATCH_SIZE
-// raise mongo timeout to 1hr if otherwise unspecified
-process.env.MONGO_SOCKET_TIMEOUT =
-  parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
-
-const { promiseMapWithLimit } = require('../../app/src/util/promises')
-const { batchedUpdate } = require('../helpers/batchedUpdate')
-const {
-  determineProjectHistoryType,
-  countProjects,
-} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
-
-const COUNT = {
-  V2: 0,
-  V1WithoutConversion: 0,
-  V1WithConversion: 0,
-  NoneWithoutConversion: 0,
-  NoneWithConversion: 0,
-  NoneWithTemporaryHistory: 0,
-  UpgradeFailed: 0,
-  ConversionFailed: 0,
-  MigratedProjects: 0,
-  TotalProjects: 0,
-}
-
-async function processBatch(projects) {
-  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
-  console.log(COUNT)
-}
-
-async function processProject(project) {
-  const historyType = await determineProjectHistoryType(project)
-  if (VERBOSE_LOGGING) {
-    console.log(
-      `project ${
-        project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
-      } is type ${historyType}`
-    )
-  }
-  COUNT[historyType] += 1
-}
-
-async function main() {
-  const projection = {
-    _id: 1,
-    overleaf: 1,
-  }
-  const options = {}
-  if (USE_QUERY_HINT) {
-    options.hint = { _id: 1 }
-  }
-  if (VERBOSE_PROJECT_NAMES) {
-    projection.name = 1
-  }
-  await batchedUpdate(
-    'projects',
-    { 'overleaf.history.display': { $ne: true } },
-    processBatch,
-    projection,
-    options
-  )
-  COUNT.MigratedProjects = await countProjects({
-    'overleaf.history.display': true,
-  })
-  COUNT.TotalProjects = await countProjects()
-  console.log('Final')
-  console.log(COUNT)
-}
-
-main()
-  .then(() => {
-    console.error('Done.')
-    process.exit(0)
-  })
-  .catch(error => {
-    console.error({ error })
-    process.exit(1)
-  })

services/web/scripts/history/debug_history.js

@@ -1,328 +0,0 @@
-// Script to debug the track-changes history of the documents in a project.
-// Usage:
-//   node debug_history.js --project-id=<project_id>
-//   node debug_history.js --all   # to check all unmigrated projects
-//
-// Example output:
-//   $ node scripts/debug_history.js  --project-id=63ff3adc06177192f18a6b38
-//   Using default settings from /overleaf/services/track-changes/config/settings.defaults.js
-//   Set UV_THREADPOOL_SIZE=16
-//   project 63ff3adc06177192f18a6b38 docId 63ff3adc06177192f18a6b3d OK
-//   project 63ff3adc06177192f18a6b38 docId 63ff3b08de41e3b0989c1720 FAILED
-//   {"action":"rewinding","version":7,"meta":{"start_ts":1677671465447,"end_ts":1677671465447,"user_id":"632ae106f9a6dd002505765b"},
-//   "ops":[{"action":"rewindOp","contentLength":24,"op":{"p":32,"d":6},"errors":[{"message":"invalid offset rewinding delete,
-//   truncating to content length","op":{"p":32,"d":6},"contentLength":24}]}],"status":"failed"}
-
-/* eslint-disable camelcase */
-const TrackChangesMongoDb = require('../../../track-changes/app/js/mongodb')
-const { waitForDb } = require('../../app/src/infrastructure/mongodb')
-const {
-  findProjects,
-} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
-const PackManager = require('../../../track-changes/app/js/PackManager')
-const {
-  packsAreDuplicated,
-} = require('../../../track-changes/app/js/util/PackUtils')
-const {
-  ConsistencyError,
-} = require('../../../track-changes/app/js/DiffGenerator')
-const DocumentUpdaterManager = require('../../../track-changes/app/js/DocumentUpdaterManager')
-const DocstoreManager = require('../../../track-changes/app/js/DocstoreManager')
-const Errors = require('../../../track-changes/app/js/Errors')
-const minimist = require('minimist')
-const util = require('util')
-const logger = require('@overleaf/logger')
-logger.initialize('debug-history')
-// disable logging to stdout from internal modules
-logger.logger.streams = []
-
-const options = {
-  boolean: ['all', 'verbose', 'raw', 'help'],
-  string: ['project-id'],
-  alias: {
-    'project-id': 'p',
-    verbose: 'v',
-    raw: 'r',
-    help: 'h',
-    all: 'a',
-  },
-  default: {},
-}
-const argv = minimist(process.argv.slice(2), options)
-
-function usage() {
-  console.log(
-    `Usage: ${process.argv[1]} [--project-id=<project_id> | --all] [--verbose] [--raw]`
-  )
-  process.exit(1)
-}
-
-// look in docstore or docupdater for the latest version of the document
-async function getLatestContent(projectId, docId, lastUpdateVersion) {
-  const [docstoreContent, docstoreVersion] =
-    await DocstoreManager.promises.peekDocument(projectId, docId)
-
-  // if docstore is out of date, check for a newer version in docupdater
-  // and return that instead
-  if (docstoreVersion <= lastUpdateVersion) {
-    const [docupdaterContent, docupdaterVersion] =
-      await DocumentUpdaterManager.promises.peekDocument(projectId, docId)
-    if (docupdaterVersion > docstoreVersion) {
-      return [docupdaterContent, docupdaterVersion]
-    }
-  }
-
-  return [docstoreContent, docstoreVersion]
-}
-
-// This class is used to write a record of all the operations that have been applied to a document
-class LogAppliedOps {
-  constructor() {
-    this.result = []
-  }
-
-  // used to log the initial state of the document
-  start(action, latestContent, version) {
-    this.result.push({
-      action,
-      latestContentLength: latestContent.length,
-      latestContent: argv.raw ? latestContent : undefined,
-      version,
-    })
-  }
-
-  // used to log a new document update
-  update(action, update) {
-    this._finalize()
-    this.opResults = []
-    this.currentResult = {
-      action,
-      version: update.v,
-      meta: update.meta,
-      ops: this.opResults,
-    }
-    this.result.push(this.currentResult)
-  }
-
-  // used to log an operation that has been applied to the document
-  op(action, content, op) {
-    this.currentOp = {
-      action,
-      contentLength: content.length,
-      content: argv.raw ? content : undefined,
-      op: this._filterOp(op),
-    }
-    this.opResults.push(this.currentOp)
-  }
-
-  // used to log an error that occurred while applying an operation
-  opError(message, content, op, err) {
-    this.currentOp.errors = this.currentOp.errors || []
-    this.currentOp.errors.push({
-      message,
-      op: this._filterOp(op),
-      contentLength: content.length,
-      content: argv.raw ? content : undefined,
-      err,
-    })
-  }
-
-  // sets the status of the current update to 'success' or 'failed'
-  // depending on whether any errors were logged
-  _finalize() {
-    if (!this.currentResult) {
-      return
-    }
-    const errors = this.opResults.some(op => op.errors)
-    this.currentResult.status = errors ? 'failed' : 'success'
-  }
-
-  // returns the final result of the log
-  end() {
-    this._finalize()
-    return this.result
-  }
-
-  // Returns a new object with the same keys as op, but with the i and d
-  // fields replaced by their lengths when present. This is used to filter
-  // out the contents of the i and d fields of an operation, to redact
-  // document content.
-  _filterOp(op) {
-    const newOp = {}
-    for (const key of Object.keys(op)) {
-      if (!argv.raw && (key === 'i' || key === 'd')) {
-        newOp[key] = op[key].length
-      } else {
-        newOp[key] = op[key]
-      }
-    }
-    return newOp
-  }
-}
-
-// This is the rewindOp function from track-changes, modified to log
-// the operation and any errors.
-function rewindOp(content, op, log) {
-  if (op.i != null) {
-    // ShareJS will accept an op where p > content.length when applied,
-    // and it applies as though p == content.length. However, the op is
-    // passed to us with the original p > content.length. Detect if that
-    // is the case with this op, and shift p back appropriately to match
-    // ShareJS if so.
-    let { p } = op
-    const maxP = content.length - op.i.length
-    if (p > maxP) {
-      log.opError(
-        'invalid offset rewinding insert, truncating to content length',
-        content,
-        op
-      )
-      p = maxP
-    }
-    const textToBeRemoved = content.slice(p, p + op.i.length)
-    if (op.i !== textToBeRemoved) {
-      log.opError(
-        'inserted content does not match text to be removed',
-        content,
-        op
-      )
-      throw new ConsistencyError(
-        `Inserted content, '${op.i}', does not match text to be removed, '${textToBeRemoved}'`
-      )
-    }
-    return content.slice(0, p) + content.slice(p + op.i.length)
-  } else if (op.d != null) {
-    if (op.p > content.length) {
-      log.opError(
-        'invalid offset rewinding delete, truncating to content length',
-        content,
-        op
-      )
-    }
-    return content.slice(0, op.p) + op.d + content.slice(op.p)
-  } else {
-    return content
-  }
-}
-
-// This is the rewindDoc function from track-changes, modified to log all
-// operations that are applied to the document.
-async function rewindDoc(projectId, docId) {
-  const log = new LogAppliedOps()
-  // Prepare to rewind content
-  const docIterator = await PackManager.promises.makeDocIterator(docId)
-  const getUpdate = util.promisify(docIterator.next).bind(docIterator)
-
-  const lastUpdate = await getUpdate()
-  if (!lastUpdate) {
-    return null
-  }
-
-  const lastUpdateVersion = lastUpdate.v
-
-  let latestContent
-  let version
-  try {
-    ;[latestContent, version] = await getLatestContent(
-      projectId,
-      docId,
-      lastUpdateVersion
-    )
-  } catch (err) {
-    if (err instanceof Errors.NotFoundError) {
-      // Doc not found in docstore. We can't build its history
-      return null
-    } else {
-      throw err
-    }
-  }
-  log.start('load-doc', latestContent, version)
-
-  let content = latestContent
-  let update = lastUpdate
-  let previousUpdate = null
-
-  while (update) {
-    if (packsAreDuplicated(update, previousUpdate)) {
-      previousUpdate = update
-      update = await getUpdate()
-      continue
-    }
-    log.update('rewinding', update)
-    for (let i = update.op.length - 1; i >= 0; i--) {
-      const op = update.op[i]
-      if (op.broken === true) {
-        log.op('skipped', content, op)
-        continue
-      }
-      try {
-        log.op('rewindOp', content, op)
-        content = rewindOp(content, op, log)
-      } catch (e) {
-        if (e instanceof ConsistencyError && (i = update.op.length - 1)) {
-          // catch known case where the last op in an array has been
-          // merged into a later op
-          op.broken = true
-          log.opError('marking broken', content, op)
-        } else {
-          log.opError('failed', content, op, e)
-        }
-      }
-    }
-    previousUpdate = update
-    update = await getUpdate()
-  }
-  return log.end()
-}
-
-async function main() {
-  // Get a list of projects to migrate
-  let projectIds = []
-  if (argv.all) {
-    const projectsToMigrate = await findProjects(
-      { 'overleaf.history.display': { $ne: true } },
-      { _id: 1, overleaf: 1 }
-    )
-    projectIds = projectsToMigrate.map(p => p._id.toString())
-    console.log('Unmigrated projects', projectIds.length)
-  } else if (argv['project-id']) {
-    projectIds = [argv['project-id']]
-  } else {
-    usage()
-    process.exit(1)
-  }
-  let errorCount = 0
-  for (const projectId of projectIds) {
-    const docIds = await PackManager.promises.findAllDocsInProject(projectId)
-    if (!docIds.length) {
-      console.log('No docs found for project', projectId)
-    }
-    let projectErrorCount = 0
-    for (const docId of docIds) {
-      const result = await rewindDoc(projectId, docId)
-      const failed = result.filter(r => r.status === 'failed')
-      errorCount += failed.length
-      if (argv.verbose) {
-        console.log(JSON.stringify({ projectId, docId, result }, null, 2))
-      } else if (failed.length > 0) {
-        console.log('project', projectId, 'docId', docId, 'FAILED')
-        for (const f of failed) {
-          console.log(JSON.stringify(f))
-        }
-        projectErrorCount += failed.length
-      }
-    }
-    if (projectErrorCount === 0 && !argv.verbose) {
-      console.log('project', projectId, 'docs', docIds.length, 'OK')
-    }
-  }
-  process.exit(errorCount > 0 ? 1 : 0)
-}
-
-waitForDb()
-  .then(TrackChangesMongoDb.waitForDb)
-  .then(main)
-  .catch(err => {
-    console.error(err)
-    process.exit(1)
-  })

services/web/scripts/history/downgrade_project.js

@@ -1,81 +0,0 @@
-const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
-const DRY_RUN = process.env.DRY_RUN !== 'false'
-process.env.MONGO_SOCKET_TIMEOUT =
-  parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
-
-const PROJECT_ID = process.env.PROJECT_ID
-
-const { ObjectId } = require('mongodb')
-const {
-  db,
-  waitForDb,
-  READ_PREFERENCE_SECONDARY,
-} = require('../../app/src/infrastructure/mongodb')
-const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
-
-console.log({
-  DRY_RUN,
-  VERBOSE_LOGGING,
-  PROJECT_ID,
-})
-
-let INTERRUPT = false
-
-async function processProject(project) {
-  if (INTERRUPT) {
-    return
-  }
-  if (!shouldPreserveHistory(project)) {
-    console.log(
-      `project ${project._id} does not have preserveHistory:true, skipping`
-    )
-    return
-  }
-  if (!DRY_RUN) {
-    await ProjectHistoryHandler.promises.downgradeHistory(project._id)
-  }
-  if (VERBOSE_LOGGING) {
-    console.log(`project ${project._id} downgraded to track-changes`)
-  }
-}
-
-async function shouldPreserveHistory(project) {
-  return await db.projectHistoryMetaData.findOne(
-    {
-      $and: [
-        { project_id: { $eq: project._id } },
-        { preserveHistory: { $eq: true } },
-      ],
-    },
-    { readPreference: READ_PREFERENCE_SECONDARY }
-  )
-}
-
-async function main() {
-  if (PROJECT_ID) {
-    await waitForDb()
-    const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) })
-    await processProject(project)
-  } else {
-    console.log('PROJECT_ID environment value is needed.')
-    process.exit(1)
-  }
-}
-
-// Upgrading history is not atomic, if we quit out mid-initialisation
-// then history could get into a broken state
-// Instead, skip any unprocessed projects and exit() at end of the batch.
-process.on('SIGINT', function () {
-  console.log('Caught SIGINT, waiting for in process downgrades to complete')
-  INTERRUPT = true
-})
-
-main()
-  .then(() => {
-    console.error('Done.')
-    process.exit(0)
-  })
-  .catch(error => {
-    console.error({ error })
-    process.exit(1)
-  })

services/web/scripts/history/migrate_history.js

@@ -1,287 +0,0 @@
-// raise mongo timeout to 1hr if otherwise unspecified
-process.env.MONGO_SOCKET_TIMEOUT =
-  parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
-
-const fs = require('fs')
-
-if (fs.existsSync('/etc/container_environment.json')) {
-  try {
-    const envData = JSON.parse(
-      fs.readFileSync('/etc/container_environment.json', 'utf8')
-    )
-    for (const [key, value] of Object.entries(envData)) {
-      process.env[key] = value
-    }
-  } catch (err) {
-    console.error(
-      'cannot read /etc/container_environment.json, the script needs to be run as root',
-      err
-    )
-    process.exit(1)
-  }
-}
-
-const VERSION = '0.9.0-cli'
-const {
-  countProjects,
-  countDocHistory,
-  upgradeProject,
-  findProjects,
-} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
-const { waitForDb } = require('../../app/src/infrastructure/mongodb')
-const minimist = require('minimist')
-const util = require('util')
-const pLimit = require('p-limit')
-const logger = require('@overleaf/logger')
-logger.initialize('history-migration')
-// disable logging to stdout from internal modules
-logger.logger.streams = []
-
-const DEFAULT_OUTPUT_FILE = `history-migration-${new Date()
-  .toISOString()
-  .replace(/[:.]/g, '_')}.log`
-
-const argv = minimist(process.argv.slice(2), {
-  boolean: [
-    'verbose',
-    'fix-invalid-characters',
-    'convert-large-docs-to-file',
-    'import-broken-history-as-zip',
-    'force-upgrade-on-failure',
-    'dry-run',
-    'use-query-hint',
-    'retry-failed',
-    'archive-on-failure',
-    'force-clean',
-  ],
-  string: ['output', 'user-id'],
-  alias: {
-    verbose: 'v',
-    output: 'o',
-    'dry-run': 'd',
-    concurrency: 'j',
-    'use-query-hint': 'q',
-    'retry-failed': 'r',
-    'archive-on-failure': 'a',
-  },
-  default: {
-    output: DEFAULT_OUTPUT_FILE,
-    concurrency: 1,
-    'batch-size': 100,
-    'max-upgrades-to-attempt': false,
-    'max-failures': 50,
-  },
-})
-
-let INTERRUPT = false
-
-async function findProjectsToMigrate() {
-  console.log('History Migration Statistics')
-
-  // Show statistics about the number of projects to migrate
-  const migratedProjects = await countProjects({
-    'overleaf.history.display': true,
-  })
-  const totalProjects = await countProjects()
-  console.log('Migrated Projects  : ', migratedProjects)
-  console.log('Total Projects     : ', totalProjects)
-  console.log('Remaining Projects : ', totalProjects - migratedProjects)
-
-  if (migratedProjects === totalProjects) {
-    console.log('All projects have been migrated')
-    process.exit(0)
-  }
-
-  // Get a list of projects to migrate
-  const projectsToMigrate = await findProjects(
-    { 'overleaf.history.display': { $ne: true } },
-    { _id: 1, overleaf: 1 }
-  )
-
-  // Show statistics for docHistory collection
-  const docHistoryWithoutProjectId = await countDocHistory({
-    project_id: { $exists: false },
-  })
-
-  if (docHistoryWithoutProjectId > 0) {
-    console.log(
-      `WARNING: docHistory collection contains ${docHistoryWithoutProjectId} records without project_id`
-    )
-    process.exit(1)
-  }
-
-  return projectsToMigrate
-}
-
-function createProgressBar() {
-  const startTime = new Date()
-  return function progressBar(current, total, msg) {
-    const barLength = 20
-    const percentage = Math.floor((current / total) * 100)
-    const bar = '='.repeat(percentage / (100 / barLength))
-    const empty = ' '.repeat(barLength - bar.length)
-    const elapsed = new Date() - startTime
-    // convert elapsed time to hours, minutes, seconds
-    const ss = Math.floor((elapsed / 1000) % 60)
-      .toString()
-      .padStart(2, '0')
-    const mm = Math.floor((elapsed / (1000 * 60)) % 60)
-      .toString()
-      .padStart(2, '0')
-    const hh = Math.floor(elapsed / (1000 * 60 * 60))
-      .toString()
-      .padStart(2, '0')
-    process.stdout.write(
-      `\r${hh}:${mm}:${ss} |${bar}${empty}| ${percentage}% (${current}/${total}) ${msg}`
-    )
-  }
-}
-
-async function migrateProjects(projectsToMigrate) {
-  let projectsMigrated = 0
-  let projectsFailed = 0
-
-  console.log('Starting migration...')
-  if (argv.concurrency > 1) {
-    console.log(`Using ${argv.concurrency} concurrent migrations`)
-  }
-  // send log output for each migration to a file
-  const output = fs.createWriteStream(argv.output, { flags: 'a' })
-  console.log(`Writing log output to ${process.cwd()}/${argv.output}`)
-  const logger = new console.Console({ stdout: output })
-  function logJson(obj) {
-    logger.log(JSON.stringify(obj))
-  }
-  // limit the number of concurrent migrations
-  const limit = pLimit(argv.concurrency)
-  const jobs = []
-  // throttle progress reporting to 2x per second
-  const progressBar = createProgressBar()
-  let i = 0
-  const N = projectsToMigrate.length
-  const progressBarTimer = setInterval(() => {
-    if (INTERRUPT) {
-      return // don't update the progress bar if we're shutting down
-    }
-    progressBar(
-      i,
-      N,
-      `Migrated: ${projectsMigrated}, Failed: ${projectsFailed}`
-    )
-  }, 500)
-
-  const options = {
-    migrationOptions: {
-      archiveOnFailure: argv['import-broken-history-as-zip'],
-      fixInvalidCharacters: argv['fix-invalid-characters'],
-      forceNewHistoryOnFailure: argv['force-upgrade-on-failure'],
-    },
-    convertLargeDocsToFile: argv['convert-large-docs-to-file'],
-    userId: argv['user-id'],
-    reason: VERSION,
-    forceClean: argv['force-clean'],
-  }
-  async function _migrateProject(project) {
-    if (INTERRUPT) {
-      return // don't start any new jobs if we're shutting down
-    }
-    const startTime = new Date()
-    try {
-      const result = await upgradeProject(project._id, options)
-      i++
-      if (INTERRUPT && limit.activeCount > 1) {
-        // an interrupt was requested while this job was running
-        // report that we're waiting for the remaining jobs to finish
-        console.log(
-          `Waiting for remaining ${
-            limit.activeCount - 1
-          } active jobs to finish\r`
-        )
-      }
-      if (result.error) {
-        // failed to migrate this project
-        logJson({
-          project_id: project._id,
-          result,
-          stack: result.error.stack,
-          startTime,
-          endTime: new Date(),
-        })
-        projectsFailed++
-      } else {
-        // successfully migrated this project
-        logJson({
-          project_id: project._id,
-          result,
-          startTime,
-          endTime: new Date(),
-        })
-        projectsMigrated++
-      }
-    } catch (err) {
-      // unexpected error from the migration
-      projectsFailed++
-      logJson({
-        project_id: project._id,
-        exception: util.inspect(err),
-        startTime,
-        endTime: new Date(),
-      })
-    }
-  }
-
-  for (const project of projectsToMigrate) {
-    jobs.push(limit(_migrateProject, project))
-  }
-  // wait for all the queued jobs to complete
-  await Promise.all(jobs)
-  clearInterval(progressBarTimer)
-  progressBar(i, N, `Migrated: ${projectsMigrated}, Failed: ${projectsFailed}`)
-  process.stdout.write('\n')
-  return { projectsMigrated, projectsFailed }
-}
-
-async function main() {
-  const projectsToMigrate = await findProjectsToMigrate()
-  if (argv['dry-run']) {
-    console.log('Dry run, exiting')
-    process.exit(0)
-  }
-  const { projectsMigrated, projectsFailed } = await migrateProjects(
-    projectsToMigrate
-  )
-  console.log('Projects migrated: ', projectsMigrated)
-  console.log('Projects failed: ', projectsFailed)
-  if (projectsFailed > 0) {
-    console.log('------------------------------------------------------')
-    console.log(`Log output written to ${process.cwd()}/${argv.output}`)
-    console.log(
-      'Please check the log for errors. Attach the content of the file when contacting support.'
-    )
-    console.log('------------------------------------------------------')
-  }
-  if (INTERRUPT) {
-    console.log('Migration interrupted, please run again to continue.')
-  } else if (projectsFailed === 0) {
-    console.log(`All projects migrated successfully.`)
-  }
-  console.log('Done.')
-  process.exit(projectsFailed > 0 ? 1 : 0)
-}
-
-// Upgrading history is not atomic, if we quit out mid-initialisation
-// then history could get into a broken state
-// Instead, skip any unprocessed projects and exit() at end of the batch.
-process.on('SIGINT', function () {
-  console.log(
-    '\nCaught SIGINT, waiting for all in-progess upgrades to complete'
-  )
-  INTERRUPT = true
-})
-
-waitForDb()
-  .then(main)
-  .catch(err => {
-    console.error(err)
-    process.exit(1)
-  })

services/web/scripts/history/reset_incorrect_doc_revision.js

@@ -1,111 +0,0 @@
-const DRY_RUN = process.env.DRY_RUN !== 'false'
-const PROJECT_ID = process.env.PROJECT_ID
-const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
-const VERBOSE_PROJECT_NAMES = process.env.VERBOSE_PROJECT_NAMES === 'true'
-const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 50
-const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 500
-// persist fallback in order to keep batchedUpdate in-sync
-process.env.BATCH_SIZE = BATCH_SIZE
-process.env.VERBOSE_LOGGING = VERBOSE_LOGGING
-
-const { ObjectId } = require('mongodb')
-const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
-const { batchedUpdate } = require('../helpers/batchedUpdate')
-const { promiseMapWithLimit } = require('../../app/src/util/promises')
-
-const count = {
-  projects: 0,
-  projectsWithIncorrectRevDocs: 0,
-  totalIncorrectRevDocs: 0,
-  totalNanRevDocs: 0,
-  totalNullRevDocs: 0,
-  totalUndefinedRevDocs: 0,
-  convertedRevs: 0,
-}
-
-async function main() {
-  const projection = {
-    _id: 1,
-  }
-
-  if (VERBOSE_PROJECT_NAMES) {
-    projection.name = 1
-  }
-
-  const options = {}
-
-  if (PROJECT_ID) {
-    const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) })
-    await processProject(project)
-  } else {
-    await batchedUpdate(
-      'projects',
-      { 'overleaf.history.display': { $ne: true } },
-      processBatch,
-      projection,
-      options
-    )
-  }
-  console.log('Final')
-}
-
-async function processBatch(projects) {
-  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
-}
-
-async function processProject(project) {
-  count.projects++
-
-  const docs = await db.docs
-    .find(
-      {
-        project_id: project._id,
-        $or: [{ rev: null }, { rev: NaN }],
-      },
-      { _id: 1, rev: 1 }
-    )
-    .toArray()
-
-  if (!docs || docs.length <= 0) {
-    return
-  }
-
-  if (VERBOSE_LOGGING) {
-    console.log(
-      `Found ${docs.length} incorrect doc.rev for project ${
-        project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
-      }`
-    )
-  }
-
-  count.projectsWithIncorrectRevDocs++
-  count.totalIncorrectRevDocs += docs.length
-
-  for (const doc of docs) {
-    if (doc.rev === undefined) {
-      count.totalUndefinedRevDocs++
-    } else if (doc.rev === null) {
-      count.totalNullRevDocs++
-    } else if (isNaN(doc.rev)) {
-      count.totalNanRevDocs++
-    } else {
-      console.error(`unknown 'rev' value: ${doc.rev}`)
-    }
-    if (!DRY_RUN) {
-      console.log(`fixing rev of doc ${doc.id} from '${doc.rev}' to 0`)
-      await db.docs.updateOne({ _id: doc._id }, { $set: { rev: 0 } })
-      count.convertedRevs++
-    }
-  }
-}
-
-waitForDb()
-  .then(main)
-  .then(() => {
-    console.log(count)
-    process.exit(0)
-  })
-  .catch(err => {
-    console.log('Something went wrong!', err)
-    process.exit(1)
-  })

services/web/scripts/history/unset_allow_downgrade.js

@@ -1,133 +0,0 @@
-const { promisify } = require('util')
-const { ObjectId } = require('mongodb')
-const {
-  db,
-  waitForDb,
-  READ_PREFERENCE_SECONDARY,
-} = require('../../app/src/infrastructure/mongodb')
-const sleep = promisify(setTimeout)
-const _ = require('lodash')
-
-const NOW_IN_S = Date.now() / 1000
-const ONE_WEEK_IN_S = 60 * 60 * 24 * 7
-const TEN_SECONDS = 10 * 1000
-
-function getSecondsFromObjectId(id) {
-  return id.getTimestamp().getTime() / 1000
-}
-
-async function main(options) {
-  if (!options) {
-    options = {}
-  }
-  _.defaults(options, {
-    projectId: process.env.PROJECT_ID,
-    dryRun: process.env.DRY_RUN !== 'false',
-    verboseLogging: process.env.VERBOSE_LOGGING === 'true',
-    firstProjectId: process.env.FIRST_PROJECT_ID
-      ? ObjectId(process.env.FIRST_PROJECT_ID)
-      : ObjectId('4b3d3b3d0000000000000000'), // timestamped to 2010-01-01T00:01:01.000Z
-    incrementByS: parseInt(process.env.INCREMENT_BY_S, 10) || ONE_WEEK_IN_S,
-    batchSize: parseInt(process.env.BATCH_SIZE, 10) || 1000,
-    stopAtS: parseInt(process.env.STOP_AT_S, 10) || NOW_IN_S,
-    letUserDoubleCheckInputsFor:
-      parseInt(process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR, 10) || TEN_SECONDS,
-  })
-
-  if (options.projectId) {
-    await waitForDb()
-    const { modifiedCount } = await db.projects.updateOne(
-      {
-        _id: ObjectId(options.projectId),
-        'overleaf.history.allowDowngrade': true,
-      },
-      { $unset: { 'overleaf.history.allowDowngrade': 1 } }
-    )
-    console.log(`modifiedCount: ${modifiedCount}`)
-    process.exit(0)
-  }
-
-  await letUserDoubleCheckInputs(options)
-  await waitForDb()
-
-  let startId = options.firstProjectId
-
-  let totalProcessed = 0
-  while (getSecondsFromObjectId(startId) <= options.stopAtS) {
-    let batchProcessed = 0
-    const end = getSecondsFromObjectId(startId) + options.incrementByS
-    let endId = ObjectId.createFromTime(end)
-    const query = {
-      _id: {
-        // include edge
-        $gte: startId,
-        // exclude edge
-        $lt: endId,
-      },
-      'overleaf.history.allowDowngrade': true,
-    }
-    const projects = await db.projects
-      .find(query, { readPreference: READ_PREFERENCE_SECONDARY })
-      .project({ _id: 1 })
-      .limit(options.batchSize)
-      .toArray()
-
-    if (projects.length) {
-      const projectIds = projects.map(project => project._id)
-      if (options.verboseLogging) {
-        console.log(
-          `Processing projects with ids: ${JSON.stringify(projectIds)}`
-        )
-      } else {
-        console.log(`Processing ${projects.length} projects`)
-      }
-
-      if (!options.dryRun) {
-        await db.projects.updateMany(
-          { _id: { $in: projectIds } },
-          { $unset: { 'overleaf.history.allowDowngrade': 1 } }
-        )
-      } else {
-        console.log(
-          `skipping update of ${projectIds.length} projects in dry-run mode`
-        )
-      }
-
-      totalProcessed += projectIds.length
-      batchProcessed += projectIds.length
-
-      if (projects.length === options.batchSize) {
-        endId = projects[projects.length - 1]._id
-      }
-    }
-    console.error(
-      `Processed ${batchProcessed} from ${startId} until ${endId} (${totalProcessed} processed in total)`
-    )
-
-    startId = endId
-  }
-}
-
-async function letUserDoubleCheckInputs(options) {
-  console.error('Options:', JSON.stringify(options, null, 2))
-  console.error(
-    'Waiting for you to double check inputs for',
-    options.letUserDoubleCheckInputsFor,
-    'ms'
-  )
-  await sleep(options.letUserDoubleCheckInputsFor)
-}
-
-module.exports = main
-
-if (require.main === module) {
-  main()
-    .then(() => {
-      console.error('Done.')
-      process.exit(0)
-    })
-    .catch(error => {
-      console.error({ error })
-      process.exit(1)
-    })
-}

services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js

@@ -1,208 +0,0 @@
-const SCRIPT_VERSION = 4
-const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
-const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
-const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
-const DRY_RUN = process.env.DRY_RUN !== 'false'
-const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
-const RETRY_FAILED = process.env.RETRY_FAILED === 'true'
-const MAX_UPGRADES_TO_ATTEMPT =
-  parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false
-const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50
-const ARCHIVE_ON_FAILURE = process.env.ARCHIVE_ON_FAILURE === 'true'
-const FIX_INVALID_CHARACTERS = process.env.FIX_INVALID_CHARACTERS === 'true'
-const FORCE_NEW_HISTORY_ON_FAILURE =
-  process.env.FORCE_NEW_HISTORY_ON_FAILURE === 'true'
-const IMPORT_ZIP_FILE_PATH = process.env.IMPORT_ZIP_FILE_PATH
-const CUTOFF_DATE = process.env.CUTOFF_DATE
-  ? new Date(process.env.CUTOFF_DATE)
-  : undefined
-// persist fallback in order to keep batchedUpdate in-sync
-process.env.BATCH_SIZE = BATCH_SIZE
-// raise mongo timeout to 1hr if otherwise unspecified
-process.env.MONGO_SOCKET_TIMEOUT =
-  parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
-
-const PROJECT_ID = process.env.PROJECT_ID
-
-// User id is required to move large documents to filestore
-const USER_ID = process.env.USER_ID
-const CONVERT_LARGE_DOCS_TO_FILE =
-  process.env.CONVERT_LARGE_DOCS_TO_FILE === 'true'
-
-const { ObjectId } = require('mongodb')
-const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
-const { promiseMapWithLimit } = require('../../app/src/util/promises')
-const { batchedUpdate } = require('../helpers/batchedUpdate')
-const {
-  anyDocHistoryExists,
-  anyDocHistoryIndexExists,
-  doUpgradeForNoneWithConversion,
-} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
-
-console.log({
-  DRY_RUN,
-  VERBOSE_LOGGING,
-  WRITE_CONCURRENCY,
-  BATCH_SIZE,
-  MAX_UPGRADES_TO_ATTEMPT,
-  MAX_FAILURES,
-  USE_QUERY_HINT,
-  RETRY_FAILED,
-  ARCHIVE_ON_FAILURE,
-  PROJECT_ID,
-  FIX_INVALID_CHARACTERS,
-  FORCE_NEW_HISTORY_ON_FAILURE,
-  CONVERT_LARGE_DOCS_TO_FILE,
-  USER_ID,
-  IMPORT_ZIP_FILE_PATH,
-  CUTOFF_DATE,
-})
-
-const RESULT = {
-  DRY_RUN,
-  attempted: 0,
-  projectsUpgraded: 0,
-  failed: 0,
-  continueFrom: null,
-}
-
-let INTERRUPT = false
-
-async function processBatch(projects) {
-  if (projects.length && projects[0]._id) {
-    RESULT.continueFrom = projects[0]._id
-  }
-  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
-  console.log(RESULT)
-  if (INTERRUPT) {
-    // ctrl+c
-    console.log('Terminated by SIGINT')
-    process.exit(0)
-  }
-  if (RESULT.failed >= MAX_FAILURES) {
-    console.log(`MAX_FAILURES limit (${MAX_FAILURES}) reached. Stopping.`)
-    process.exit(0)
-  }
-  if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
-    console.log(
-      `MAX_UPGRADES_TO_ATTEMPT limit (${MAX_UPGRADES_TO_ATTEMPT}) reached. Stopping.`
-    )
-    process.exit(0)
-  }
-}
-
-async function processProject(project) {
-  if (INTERRUPT) {
-    return
-  }
-  if (project.overleaf && project.overleaf.history) {
-    // projects we're upgrading like this should never have a history id
-    if (project.overleaf.history.id) {
-      return
-    }
-    if (
-      project.overleaf.history.conversionFailed ||
-      project.overleaf.history.upgradeFailed
-    ) {
-      if (project.overleaf.history.zipFileArchivedInProject) {
-        return // always give up if we have uploaded the zipfile to the project
-      }
-      if (!RETRY_FAILED) {
-        // we don't want to attempt upgrade on projects
-        // that have been previously attempted and failed
-        return
-      }
-    }
-  }
-  if (RESULT.failed >= MAX_FAILURES) {
-    return
-  }
-  if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
-    return
-  }
-  const anyDocHistoryOrIndex =
-    (await anyDocHistoryExists(project)) ||
-    (await anyDocHistoryIndexExists(project))
-  if (anyDocHistoryOrIndex) {
-    RESULT.attempted += 1
-    if (DRY_RUN) {
-      return
-    }
-    const result = await doUpgradeForNoneWithConversion(project, {
-      migrationOptions: {
-        archiveOnFailure: ARCHIVE_ON_FAILURE,
-        fixInvalidCharacters: FIX_INVALID_CHARACTERS,
-        forceNewHistoryOnFailure: FORCE_NEW_HISTORY_ON_FAILURE,
-        importZipFilePath: IMPORT_ZIP_FILE_PATH,
-        cutoffDate: CUTOFF_DATE,
-      },
-      convertLargeDocsToFile: CONVERT_LARGE_DOCS_TO_FILE,
-      userId: USER_ID,
-      reason: `${SCRIPT_VERSION}`,
-    })
-    if (result.convertedDocCount) {
-      console.log(
-        `project ${project._id} converted ${result.convertedDocCount} docs to filestore`
-      )
-    }
-    if (result.error) {
-      console.error(`project ${project._id} FAILED with error: `, result.error)
-      RESULT.failed += 1
-    } else if (result.upgraded) {
-      if (VERBOSE_LOGGING) {
-        console.log(
-          `project ${project._id} converted and upgraded to full project history`
-        )
-      }
-      RESULT.projectsUpgraded += 1
-    }
-  }
-}
-
-async function main() {
-  if (PROJECT_ID) {
-    await waitForDb()
-    const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) })
-    await processProject(project)
-  } else {
-    const projection = {
-      _id: 1,
-      overleaf: 1,
-    }
-    const options = {}
-    if (USE_QUERY_HINT) {
-      options.hint = { _id: 1 }
-    }
-    await batchedUpdate(
-      'projects',
-      // we originally used
-      // 'overleaf.history.id': { $exists: false }
-      // but display false is indexed and contains all the above,
-      // it can be faster to skip projects with a history ID than to use a query
-      { 'overleaf.history.display': { $ne: true } },
-      processBatch,
-      projection,
-      options
-    )
-  }
-  console.log('Final')
-  console.log(RESULT)
-}
-
-// Upgrading history is not atomic, if we quit out mid-initialisation
-// then history could get into a broken state
-// Instead, skip any unprocessed projects and exit() at end of the batch.
-process.on('SIGINT', function () {
-  console.log('Caught SIGINT, waiting for in process upgrades to complete')
-  INTERRUPT = true
-})
-
-main()
-  .then(() => {
-    console.error('Done.')
-    process.exit(0)
-  })
-  .catch(error => {
-    console.error({ error })
-    process.exit(1)
-  })

services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js

@@ -1,232 +0,0 @@
-const SCRIPT_VERSION = 3
-const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
-const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
-const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
-const DRY_RUN = process.env.DRY_RUN !== 'false'
-const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
-const RETRY_FAILED = process.env.RETRY_FAILED === 'true'
-const MAX_UPGRADES_TO_ATTEMPT =
-  parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false
-const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50
-// persist fallback in order to keep batchedUpdate in-sync
-process.env.BATCH_SIZE = BATCH_SIZE
-// raise mongo timeout to 1hr if otherwise unspecified
-process.env.MONGO_SOCKET_TIMEOUT =
-  parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
-
-const {
-  db,
-  READ_PREFERENCE_SECONDARY,
-} = require('../../app/src/infrastructure/mongodb')
-const { promiseMapWithLimit } = require('../../app/src/util/promises')
-const { batchedUpdate } = require('../helpers/batchedUpdate')
-const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
-const HistoryManager = require('../../app/src/Features/History/HistoryManager')
-
-console.log({
-  DRY_RUN,
-  VERBOSE_LOGGING,
-  WRITE_CONCURRENCY,
-  BATCH_SIZE,
-  MAX_UPGRADES_TO_ATTEMPT,
-  MAX_FAILURES,
-  USE_QUERY_HINT,
-  RETRY_FAILED,
-})
-
-const RESULT = {
-  DRY_RUN,
-  attempted: 0,
-  projectsUpgraded: 0,
-  failed: 0,
-  continueFrom: null,
-}
-
-let INTERRUPT = false
-
-async function processBatch(projects) {
-  if (projects.length && projects[0]._id) {
-    RESULT.continueFrom = projects[0]._id
-  }
-  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
-  console.log(RESULT)
-  if (INTERRUPT) {
-    // ctrl+c
-    console.log('Terminated by SIGINT')
-    process.exit(0)
-  }
-  if (RESULT.failed >= MAX_FAILURES) {
-    console.log(`MAX_FAILURES limit (${MAX_FAILURES}) reached. Stopping.`)
-    process.exit(0)
-  }
-  if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
-    console.log(
-      `MAX_UPGRADES_TO_ATTEMPT limit (${MAX_UPGRADES_TO_ATTEMPT}) reached. Stopping.`
-    )
-    process.exit(0)
-  }
-}
-
-async function processProject(project) {
-  if (INTERRUPT) {
-    return
-  }
-  // If upgradeFailed, skip unless we're explicitly retrying failed upgrades
-  if (
-    project.overleaf &&
-    project.overleaf.history &&
-    project.overleaf.history.upgradeFailed
-  ) {
-    if (RETRY_FAILED) {
-      return await doUpgradeForNoneWithoutConversion(project)
-    } else {
-      return
-    }
-  }
-  // Skip any projects with a history ID, these are v1
-  if (
-    project.overleaf &&
-    project.overleaf.history &&
-    project.overleaf.history.id
-  ) {
-    return
-  }
-  const anyDocHistory = await anyDocHistoryExists(project)
-  if (anyDocHistory) {
-    return
-  }
-  const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-  if (anyDocHistoryIndex) {
-    return
-  }
-  await doUpgradeForNoneWithoutConversion(project)
-}
-
-async function doUpgradeForNoneWithoutConversion(project) {
-  if (RESULT.failed >= MAX_FAILURES) {
-    return
-  }
-  if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
-    return
-  }
-  RESULT.attempted += 1
-  const projectId = project._id
-  if (!DRY_RUN) {
-    // ensureHistoryExistsForProject resyncs project
-    // Change to 'peek'ing the doc when resyncing should
-    // be rolled out prior to using this script
-    try {
-      // Logic originally from ProjectHistoryHandler.ensureHistoryExistsForProject
-      // However sends a force resync project to project history instead
-      // of a resync request to doc-updater
-      let historyId = await ProjectHistoryHandler.promises.getHistoryId(
-        projectId
-      )
-      if (historyId == null) {
-        historyId = await HistoryManager.promises.initializeProject(projectId)
-        if (historyId != null) {
-          await ProjectHistoryHandler.promises.setHistoryId(
-            projectId,
-            historyId
-          )
-        }
-      }
-      await HistoryManager.promises.resyncProject(projectId, {
-        force: true,
-        origin: { kind: 'history-migration' },
-      })
-      await HistoryManager.promises.flushProject(projectId)
-    } catch (err) {
-      RESULT.failed += 1
-      console.error(`project ${project._id} FAILED with error: `, err)
-      await db.projects.updateOne(
-        { _id: project._id },
-        {
-          $set: {
-            'overleaf.history.upgradeFailed': true,
-          },
-        }
-      )
-      return
-    }
-    await db.projects.updateOne(
-      { _id: project._id },
-      {
-        $set: {
-          'overleaf.history.display': true,
-          'overleaf.history.upgradedAt': new Date(),
-          'overleaf.history.upgradeReason': `none-without-sl-history/${SCRIPT_VERSION}`,
-        },
-        $unset: {
-          'overleaf.history.upgradeFailed': true,
-        },
-      }
-    )
-  }
-  if (VERBOSE_LOGGING) {
-    console.log(`project ${project._id} converted to full project history`)
-  }
-  RESULT.projectsUpgraded += 1
-}
-
-async function anyDocHistoryExists(project) {
-  return await db.docHistory.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: READ_PREFERENCE_SECONDARY,
-    }
-  )
-}
-
-async function anyDocHistoryIndexExists(project) {
-  return await db.docHistoryIndex.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: READ_PREFERENCE_SECONDARY,
-    }
-  )
-}
-
-async function main() {
-  const projection = {
-    _id: 1,
-    overleaf: 1,
-  }
-  const options = {}
-  if (USE_QUERY_HINT) {
-    options.hint = { _id: 1 }
-  }
-  await batchedUpdate(
-    'projects',
-    // we originally used
-    // 'overleaf.history.id': { $exists: false }
-    // but display false is indexed and contains all the above,
-    // plus we want to be able to retry failed upgrades with a history id
-    { 'overleaf.history.display': { $ne: true } },
-    processBatch,
-    projection,
-    options
-  )
-  console.log('Final')
-  console.log(RESULT)
-}
-
-// Upgrading history is not atomic, if we quit out mid-initialisation
-// then history could get into a broken state
-// Instead, skip any unprocessed projects and exit() at end of the batch.
-process.on('SIGINT', function () {
-  console.log('Caught SIGINT, waiting for in process upgrades to complete')
-  INTERRUPT = true
-})
-
-main()
-  .then(() => {
-    console.error('Done.')
-    process.exit(0)
-  })
-  .catch(error => {
-    console.error({ error })
-    process.exit(1)
-  })

services/web/scripts/history/upgrade_project.js

@@ -1,44 +0,0 @@
-const { ObjectId } = require('mongodb')
-const {
-  db,
-  waitForDb,
-  READ_PREFERENCE_SECONDARY,
-} = require('../../app/src/infrastructure/mongodb')
-const {
-  upgradeProject,
-} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
-
-async function processProject(project) {
-  const result = await upgradeProject(project)
-  console.log(result)
-}
-
-async function main() {
-  await waitForDb()
-  const args = process.argv.slice(2)
-  const projectId = args[0]
-  const query = { _id: ObjectId(projectId) }
-  const projection = {
-    _id: 1,
-    overleaf: 1,
-  }
-  const options = {
-    projection,
-    readPreference: READ_PREFERENCE_SECONDARY,
-  }
-  const project = await db.projects.findOne(query, options)
-  if (project) {
-    await processProject(project)
-  } else {
-    console.error(`project ${projectId} not found`)
-  }
-}
-
-main()
-  .then(() => {
-    process.exit(0)
-  })
-  .catch(error => {
-    console.error({ error })
-    process.exit(1)
-  })

services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js

@@ -1,243 +0,0 @@
-const SCRIPT_VERSION = 1
-const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
-const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
-const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
-const DRY_RUN = process.env.DRY_RUN !== 'false'
-const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
-const RETRY_FAILED = process.env.RETRY_FAILED === 'true'
-const MAX_UPGRADES_TO_ATTEMPT =
-  parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false
-const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50
-// persist fallback in order to keep batchedUpdate in-sync
-process.env.BATCH_SIZE = BATCH_SIZE
-// raise mongo timeout to 1hr if otherwise unspecified
-process.env.MONGO_SOCKET_TIMEOUT =
-  parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
-
-const PROJECT_ID = process.env.PROJECT_ID
-
-const { ObjectId } = require('mongodb')
-const {
-  db,
-  waitForDb,
-  READ_PREFERENCE_SECONDARY,
-} = require('../../app/src/infrastructure/mongodb')
-const { promiseMapWithLimit } = require('../../app/src/util/promises')
-const { batchedUpdate } = require('../helpers/batchedUpdate')
-const ProjectHistoryController = require('../../modules/history-migration/app/src/ProjectHistoryController')
-
-console.log({
-  DRY_RUN,
-  VERBOSE_LOGGING,
-  WRITE_CONCURRENCY,
-  BATCH_SIZE,
-  MAX_UPGRADES_TO_ATTEMPT,
-  MAX_FAILURES,
-  USE_QUERY_HINT,
-  RETRY_FAILED,
-  PROJECT_ID,
-})
-
-const RESULT = {
-  DRY_RUN,
-  attempted: 0,
-  projectsUpgraded: 0,
-  failed: 0,
-  continueFrom: null,
-}
-
-let INTERRUPT = false
-
-async function processBatch(projects) {
-  if (projects.length && projects[0]._id) {
-    RESULT.continueFrom = projects[0]._id
-  }
-  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
-  console.log(RESULT)
-  if (INTERRUPT) {
-    // ctrl+c
-    console.log('Terminated by SIGINT')
-    process.exit(0)
-  }
-  if (RESULT.failed >= MAX_FAILURES) {
-    console.log(`MAX_FAILURES limit (${MAX_FAILURES}) reached. Stopping.`)
-    process.exit(0)
-  }
-  if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
-    console.log(
-      `MAX_UPGRADES_TO_ATTEMPT limit (${MAX_UPGRADES_TO_ATTEMPT}) reached. Stopping.`
-    )
-    process.exit(0)
-  }
-}
-
-async function processProject(project) {
-  if (INTERRUPT) {
-    return
-  }
-  // skip safety check if we want to retry failed upgrades
-  if (!RETRY_FAILED) {
-    if (project.overleaf && project.overleaf.history) {
-      if (
-        project.overleaf.history.conversionFailed ||
-        project.overleaf.history.upgradeFailed
-      ) {
-        // we don't want to attempt upgrade on projects
-        // that have been previously attempted and failed
-        return
-      }
-    }
-  }
-  const preserveHistory = await shouldPreserveHistory(project)
-  if (preserveHistory) {
-    const anyDocHistory = await anyDocHistoryExists(project)
-    if (anyDocHistory) {
-      return await doUpgradeForV1WithConversion(project)
-    }
-    const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-    if (anyDocHistoryIndex) {
-      return await doUpgradeForV1WithConversion(project)
-    }
-  }
-}
-
-async function doUpgradeForV1WithConversion(project) {
-  if (RESULT.failed >= MAX_FAILURES) {
-    return
-  }
-  if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
-    return
-  }
-  RESULT.attempted += 1
-  const projectId = project._id
-  // migrateProjectHistory expects project id as a string
-  const projectIdString = project._id.toString()
-  if (!DRY_RUN) {
-    try {
-      // We treat these essentially as None projects, the V1 history is irrelevant,
-      // so we will delete it, and do a conversion as if we're a None project
-      await ProjectHistoryController.deleteProjectHistory(projectIdString)
-      if (VERBOSE_LOGGING) {
-        console.log(
-          `project ${projectId} existing full project history deleted`
-        )
-      }
-      await ProjectHistoryController.migrateProjectHistory(projectIdString)
-    } catch (err) {
-      // if migrateProjectHistory fails, it cleans up by deleting
-      // the history and unsetting the history id
-      // therefore a failed project will still look like a 'None with conversion' project
-      RESULT.failed += 1
-      console.error(`project ${projectId} FAILED with error: `, err)
-      // We set a failed flag so future runs of the script don't automatically retry
-      await db.projects.updateOne(
-        { _id: projectId },
-        {
-          $set: {
-            'overleaf.history.conversionFailed': true,
-          },
-        }
-      )
-      return
-    }
-    await db.projects.updateOne(
-      { _id: projectId },
-      {
-        $set: {
-          'overleaf.history.upgradeReason': `v1-with-conversion/${SCRIPT_VERSION}`,
-        },
-        $unset: {
-          'overleaf.history.upgradeFailed': true,
-          'overleaf.history.conversionFailed': true,
-        },
-      }
-    )
-  }
-  if (VERBOSE_LOGGING) {
-    console.log(
-      `project ${projectId} converted and upgraded to full project history`
-    )
-  }
-  RESULT.projectsUpgraded += 1
-}
-
-async function shouldPreserveHistory(project) {
-  return await db.projectHistoryMetaData.findOne(
-    {
-      $and: [
-        { project_id: { $eq: project._id } },
-        { preserveHistory: { $eq: true } },
-      ],
-    },
-    { readPreference: READ_PREFERENCE_SECONDARY }
-  )
-}
-
-async function anyDocHistoryExists(project) {
-  return await db.docHistory.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: READ_PREFERENCE_SECONDARY,
-    }
-  )
-}
-
-async function anyDocHistoryIndexExists(project) {
-  return await db.docHistoryIndex.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: READ_PREFERENCE_SECONDARY,
-    }
-  )
-}
-
-async function main() {
-  if (PROJECT_ID) {
-    await waitForDb()
-    const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) })
-    await processProject(project)
-  } else {
-    const projection = {
-      _id: 1,
-      overleaf: 1,
-    }
-    const options = {}
-    if (USE_QUERY_HINT) {
-      options.hint = { _id: 1 }
-    }
-    await batchedUpdate(
-      'projects',
-      {
-        $and: [
-          { 'overleaf.history.display': { $ne: true } },
-          { 'overleaf.history.id': { $exists: true } },
-        ],
-      },
-      processBatch,
-      projection,
-      options
-    )
-    console.log('Final')
-    console.log(RESULT)
-  }
-}
-
-// Upgrading history is not atomic, if we quit out mid-initialisation
-// then history could get into a broken state
-// Instead, skip any unprocessed projects and exit() at end of the batch.
-process.on('SIGINT', function () {
-  console.log('Caught SIGINT, waiting for in process upgrades to complete')
-  INTERRUPT = true
-})
-
-main()
-  .then(() => {
-    console.error('Done.')
-    process.exit(0)
-  })
-  .catch(error => {
-    console.error({ error })
-    process.exit(1)
-  })

services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js

@@ -1,165 +0,0 @@
-const SCRIPT_VERSION = 2
-const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
-const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
-const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
-const DRY_RUN = process.env.DRY_RUN !== 'false'
-const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
-// persist fallback in order to keep batchedUpdate in-sync
-process.env.BATCH_SIZE = BATCH_SIZE
-// raise mongo timeout to 1hr if otherwise unspecified
-process.env.MONGO_SOCKET_TIMEOUT =
-  parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
-
-const { ObjectId } = require('mongodb')
-const {
-  db,
-  READ_PREFERENCE_SECONDARY,
-} = require('../../app/src/infrastructure/mongodb')
-const { promiseMapWithLimit } = require('../../app/src/util/promises')
-const { batchedUpdate } = require('../helpers/batchedUpdate')
-
-console.log({
-  DRY_RUN,
-  VERBOSE_LOGGING,
-  WRITE_CONCURRENCY,
-  BATCH_SIZE,
-  USE_QUERY_HINT,
-})
-
-const RESULT = {
-  DRY_RUN,
-  projectsUpgraded: 0,
-}
-
-const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = '5a8d8a370000000000000000'
-const OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = new ObjectId(
-  ID_WHEN_FULL_PROJECT_HISTORY_ENABLED
-)
-const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED =
-  OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED.getTimestamp()
-
-// set a default BATCH_LAST_ID at our cutoff point if none set
-// we still check against this cut off point later, even if
-// BATCH_LAST_ID is set to something problematic
-if (!process.env.BATCH_LAST_ID) {
-  process.env.BATCH_LAST_ID = ID_WHEN_FULL_PROJECT_HISTORY_ENABLED
-}
-
-async function processBatch(projects) {
-  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
-  console.log(RESULT)
-}
-
-async function processProject(project) {
-  // safety check
-  if (
-    project.overleaf &&
-    project.overleaf.history &&
-    project.overleaf.history.upgradeFailed
-  ) {
-    // a failed history upgrade might look like a v1 project, but history may be broken
-    return
-  }
-  if (!projectCreatedAfterFullProjectHistoryEnabled(project)) {
-    return
-  }
-  // if they have SL history, continue to send to both history systems (for now)
-  const anyDocHistory = await anyDocHistoryExists(project)
-  if (anyDocHistory) {
-    return await doUpgradeForV1WithoutConversion(project, true)
-  }
-  const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-  if (anyDocHistoryIndex) {
-    return await doUpgradeForV1WithoutConversion(project, true)
-  }
-  // or if no sl history, nothing to 'downgrade' to
-  return await doUpgradeForV1WithoutConversion(project, false)
-}
-
-function projectCreatedAfterFullProjectHistoryEnabled(project) {
-  return (
-    project._id.getTimestamp() >= DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED
-  )
-}
-
-async function doUpgradeForV1WithoutConversion(project, allowDowngrade) {
-  const setProperties = {
-    'overleaf.history.display': true,
-    'overleaf.history.upgradedAt': new Date(),
-    'overleaf.history.upgradeReason': `v1-after-fph/${SCRIPT_VERSION}`,
-  }
-  if (allowDowngrade) {
-    setProperties['overleaf.history.allowDowngrade'] = true
-  }
-  if (!DRY_RUN) {
-    await db.projects.updateOne(
-      { _id: project._id },
-      {
-        $set: setProperties,
-      }
-    )
-  }
-  if (VERBOSE_LOGGING) {
-    console.log(
-      `project ${project._id} converted to full project history${
-        allowDowngrade ? ', with allowDowngrade' : ''
-      }`
-    )
-  }
-  RESULT.projectsUpgraded += 1
-}
-
-async function anyDocHistoryExists(project) {
-  return await db.docHistory.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: READ_PREFERENCE_SECONDARY,
-    }
-  )
-}
-
-async function anyDocHistoryIndexExists(project) {
-  return await db.docHistoryIndex.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: READ_PREFERENCE_SECONDARY,
-    }
-  )
-}
-
-async function main() {
-  const projection = {
-    _id: 1,
-    overleaf: 1,
-  }
-  const options = {}
-  if (USE_QUERY_HINT) {
-    options.hint = { _id: 1 }
-  }
-  await batchedUpdate(
-    'projects',
-    {
-      $and: [
-        { 'overleaf.history.display': { $ne: true } },
-        { 'overleaf.history.id': { $exists: true } },
-      ],
-    },
-    processBatch,
-    projection,
-    options
-  )
-  console.log('Final')
-  console.log(RESULT)
-}
-
-main()
-  .then(() => {
-    console.error('Done.')
-    process.exit(0)
-  })
-  .catch(error => {
-    console.error({ error })
-    process.exit(1)
-  })

services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js

@@ -1,161 +0,0 @@
-const SCRIPT_VERSION = 3
-const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
-const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
-const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
-const DRY_RUN = process.env.DRY_RUN !== 'false'
-const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
-const UPGRADE_FAILED_WITH_EMPTY_HISTORY =
-  process.env.UPGRADE_FAILED_WITH_EMPTY_HISTORY === 'true'
-// persist fallback in order to keep batchedUpdate in-sync
-process.env.BATCH_SIZE = BATCH_SIZE
-// raise mongo timeout to 1hr if otherwise unspecified
-process.env.MONGO_SOCKET_TIMEOUT =
-  parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
-
-const {
-  db,
-  READ_PREFERENCE_SECONDARY,
-} = require('../../app/src/infrastructure/mongodb')
-const { promiseMapWithLimit } = require('../../app/src/util/promises')
-const { batchedUpdate } = require('../helpers/batchedUpdate')
-
-console.log({
-  DRY_RUN,
-  VERBOSE_LOGGING,
-  WRITE_CONCURRENCY,
-  BATCH_SIZE,
-  USE_QUERY_HINT,
-  UPGRADE_FAILED_WITH_EMPTY_HISTORY,
-})
-
-const RESULT = {
-  DRY_RUN,
-  projectsUpgraded: 0,
-}
-
-async function processBatch(projects) {
-  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
-  console.log(RESULT)
-}
-
-async function processProject(project) {
-  // safety check if history exists and there was a failed upgrade
-  const anyDocHistory = await anyDocHistoryExists(project)
-  const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-  if (
-    project.overleaf &&
-    project.overleaf.history &&
-    project.overleaf.history.upgradeFailed
-  ) {
-    const emptyHistory = !anyDocHistory && !anyDocHistoryIndex
-    if (emptyHistory && UPGRADE_FAILED_WITH_EMPTY_HISTORY) {
-      console.log(
-        `upgrading previously failed project ${project._id} with empty history`
-      )
-    } else {
-      // a failed history upgrade might look like a v1 project, but history may be broken
-      return
-    }
-  }
-  const preserveHistory = await shouldPreserveHistory(project)
-  if (preserveHistory) {
-    // if we need to preserve history, then we must bail out if history exists
-    if (anyDocHistory) {
-      return
-    }
-    if (anyDocHistoryIndex) {
-      return
-    }
-    return await doUpgradeForV1WithoutConversion(project)
-  } else {
-    // if preserveHistory false, then max 7 days of SL history
-    // but v1 already record to both histories, so safe to upgrade
-    return await doUpgradeForV1WithoutConversion(project)
-  }
-}
-
-async function doUpgradeForV1WithoutConversion(project) {
-  if (!DRY_RUN) {
-    await db.projects.updateOne(
-      { _id: project._id },
-      {
-        $set: {
-          'overleaf.history.display': true,
-          'overleaf.history.upgradedAt': new Date(),
-          'overleaf.history.upgradeReason': `v1-without-sl-history/${SCRIPT_VERSION}`,
-        },
-      }
-    )
-  }
-  if (VERBOSE_LOGGING) {
-    console.log(`project ${project._id} converted to full project history`)
-  }
-  RESULT.projectsUpgraded += 1
-}
-
-async function shouldPreserveHistory(project) {
-  return await db.projectHistoryMetaData.findOne(
-    {
-      $and: [
-        { project_id: { $eq: project._id } },
-        { preserveHistory: { $eq: true } },
-      ],
-    },
-    { readPreference: READ_PREFERENCE_SECONDARY }
-  )
-}
-
-async function anyDocHistoryExists(project) {
-  return await db.docHistory.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: READ_PREFERENCE_SECONDARY,
-    }
-  )
-}
-
-async function anyDocHistoryIndexExists(project) {
-  return await db.docHistoryIndex.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: READ_PREFERENCE_SECONDARY,
-    }
-  )
-}
-
-async function main() {
-  const projection = {
-    _id: 1,
-    overleaf: 1,
-  }
-  const options = {}
-  if (USE_QUERY_HINT) {
-    options.hint = { _id: 1 }
-  }
-  await batchedUpdate(
-    'projects',
-    {
-      $and: [
-        { 'overleaf.history.display': { $ne: true } },
-        { 'overleaf.history.id': { $exists: true } },
-      ],
-    },
-    processBatch,
-    projection,
-    options
-  )
-  console.log('Final')
-  console.log(RESULT)
-}
-
-main()
-  .then(() => {
-    console.error('Done.')
-    process.exit(0)
-  })
-  .catch(error => {
-    console.error({ error })
-    process.exit(1)
-  })

services/web/scripts/recover_docs_from_redis.js

@@ -165,7 +165,6 @@ async function deleteDocFromRedis(projectId, docId) {
     `UnflushedTime:{${docId}}`,
     `Pathname:{${docId}}`,
     `ProjectHistoryId:{${docId}}`,
-    `ProjectHistoryType:{${docId}}`,
     `PendingUpdates:{${docId}}`,
     `lastUpdatedAt:{${docId}}`,
     `lastUpdatedBy:{${docId}}`

services/web/test/acceptance/config/settings.test.defaults.js

@@ -51,9 +51,6 @@ module.exports = {
       url: 'http://localhost:23005',
       host: 'localhost',
     },
-    trackchanges: {
-      url: 'http://localhost:23015',
-    },
     docstore: {
       url: 'http://localhost:23016',
       pubUrl: 'http://localhost:23016',
@@ -78,8 +75,6 @@ module.exports = {
     },
     project_history: {
       sendProjectStructureOps: true,
-      initializeHistoryForNewProjects: true,
-      displayHistoryForNewProjects: true,
       url: `http://localhost:23054`,
     },
     v1_history: {

services/web/test/unit/src/Documents/DocumentControllerTests.js

@@ -39,77 +39,6 @@ describe('DocumentController', function () {
       }
     })
 
-    describe('when the project exists without project history enabled', function () {
-      beforeEach(function () {
-        this.project = { _id: this.project_id }
-        this.ProjectGetter.getProject = sinon
-          .stub()
-          .callsArgWith(2, null, this.project)
-      })
-
-      describe('when the document exists', function () {
-        beforeEach(function () {
-          this.doc = { _id: this.doc_id }
-          this.ProjectLocator.findElement = sinon
-            .stub()
-            .callsArgWith(1, null, this.doc, { fileSystem: this.pathname })
-          this.ProjectEntityHandler.getDoc = sinon
-            .stub()
-            .yields(null, this.doc_lines, this.rev, this.version, this.ranges)
-          this.DocumentController.getDocument(this.req, this.res, this.next)
-        })
-
-        it('should get the project', function () {
-          this.ProjectGetter.getProject
-            .calledWith(this.project_id, { rootFolder: true, overleaf: true })
-            .should.equal(true)
-        })
-
-        it('should get the pathname of the document', function () {
-          this.ProjectLocator.findElement
-            .calledWith({
-              project: this.project,
-              element_id: this.doc_id,
-              type: 'doc',
-            })
-            .should.equal(true)
-        })
-
-        it('should get the document content', function () {
-          this.ProjectEntityHandler.getDoc
-            .calledWith(this.project_id, this.doc_id)
-            .should.equal(true)
-        })
-
-        it('should return the document data to the client as JSON', function () {
-          this.res.type.should.equal('application/json')
-          this.res.body.should.equal(
-            JSON.stringify({
-              lines: this.doc_lines,
-              version: this.version,
-              ranges: this.ranges,
-              pathname: this.pathname,
-            })
-          )
-        })
-      })
-
-      describe("when the document doesn't exist", function () {
-        beforeEach(function () {
-          this.ProjectLocator.findElement = sinon
-            .stub()
-            .callsArgWith(1, new Errors.NotFoundError('not found'))
-          this.DocumentController.getDocument(this.req, this.res, this.next)
-        })
-
-        it('should call next with the NotFoundError', function () {
-          this.next
-            .calledWith(sinon.match.instanceOf(Errors.NotFoundError))
-            .should.equal(true)
-        })
-      })
-    })
-
     describe('when project exists with project history enabled', function () {
       beforeEach(function () {
         this.doc = { _id: this.doc_id }
@@ -152,53 +81,6 @@ describe('DocumentController', function () {
       })
     })
 
-    describe('when project exists that was migrated with downgrades allowed', function () {
-      beforeEach(function () {
-        this.doc = { _id: this.doc_id }
-        this.projectHistoryId = 1234
-        this.projectHistoryDisplay = true
-        this.projectHistoryType = undefined
-        this.project = {
-          _id: this.project_id,
-          overleaf: {
-            history: {
-              id: this.projectHistoryId,
-              display: this.projectHistoryDisplay,
-              allowDowngrade: true,
-            },
-          },
-        }
-        this.ProjectGetter.getProject = sinon
-          .stub()
-          .callsArgWith(2, null, this.project)
-        this.ProjectLocator.findElement = sinon
-          .stub()
-          .callsArgWith(1, null, this.doc, { fileSystem: this.pathname })
-        this.ProjectEntityHandler.getDoc = sinon
-          .stub()
-          .yields(null, this.doc_lines, this.rev, this.version, this.ranges)
-        return this.DocumentController.getDocument(
-          this.req,
-          this.res,
-          this.next
-        )
-      })
-
-      it('should return the history id in the JSON but not history type, sending history to both services', function () {
-        this.res.type.should.equal('application/json')
-        return this.res.body.should.equal(
-          JSON.stringify({
-            lines: this.doc_lines,
-            version: this.version,
-            ranges: this.ranges,
-            pathname: this.pathname,
-            projectHistoryId: this.projectHistoryId,
-            projectHistoryType: this.projectHistoryType,
-          })
-        )
-      })
-    })
-
     describe('when the project does not exist', function () {
       beforeEach(function () {
         this.ProjectGetter.getProject = sinon.stub().callsArgWith(2, null, null)

services/web/test/unit/src/History/HistoryControllerTests.js

@@ -46,60 +46,12 @@ describe('HistoryController', function () {
       },
     })
     return (this.settings.apis = {
-      trackchanges: {
-        enabled: false,
-        url: 'http://trackchanges.example.com',
-      },
       project_history: {
         url: 'http://project_history.example.com',
       },
     })
   })
 
-  describe('selectHistoryApi', function () {
-    beforeEach(function () {
-      this.req = { url: '/mock/url', method: 'POST', params: {} }
-      this.res = 'mock-res'
-      return (this.next = sinon.stub())
-    })
-
-    describe('for a project with project history', function () {
-      beforeEach(function () {
-        this.ProjectDetailsHandler.getDetails = sinon
-          .stub()
-          .callsArgWith(1, null, {
-            overleaf: { history: { id: 42, display: true } },
-          })
-        return this.HistoryController.selectHistoryApi(
-          this.req,
-          this.res,
-          this.next
-        )
-      })
-
-      it('should set the flag for project history to true', function () {
-        return this.req.useProjectHistory.should.equal(true)
-      })
-    })
-
-    describe('for any other project ', function () {
-      beforeEach(function () {
-        this.ProjectDetailsHandler.getDetails = sinon
-          .stub()
-          .callsArgWith(1, null, {})
-        return this.HistoryController.selectHistoryApi(
-          this.req,
-          this.res,
-          this.next
-        )
-      })
-
-      it('should not set the flag for project history to false', function () {
-        return this.req.useProjectHistory.should.equal(false)
-      })
-    })
-  })
-
   describe('proxyToHistoryApi', function () {
     beforeEach(function () {
       this.req = { url: '/mock/url', method: 'POST' }
@@ -161,18 +113,6 @@ describe('HistoryController', function () {
           .should.equal(true)
       })
 
-      it('should call the track changes api', function () {
-        return this.request
-          .calledWith({
-            url: `${this.settings.apis.trackchanges.url}${this.req.url}`,
-            method: this.req.method,
-            headers: {
-              'X-User-Id': this.user_id,
-            },
-          })
-          .should.equal(true)
-      })
-
       it('should pipe the response to the client', function () {
         expect(this.Stream.pipeline).to.have.been.calledWith(
           this.proxy,
@@ -249,19 +189,6 @@ describe('HistoryController', function () {
           .should.equal(true)
       })
 
-      it('should call the track changes api', function () {
-        return this.request
-          .calledWith({
-            url: `${this.settings.apis.trackchanges.url}${this.req.url}`,
-            method: this.req.method,
-            json: true,
-            headers: {
-              'X-User-Id': this.user_id,
-            },
-          })
-          .should.equal(true)
-      })
-
       it('should inject the user data', function () {
         return this.HistoryManager.injectUserDetails
           .calledWith(this.data)

services/web/test/unit/src/History/HistoryManagerTests.js

@@ -23,10 +23,6 @@ describe('HistoryManager', function () {
     this.v1HistoryPassword = 'verysecret'
     this.settings = {
       apis: {
-        trackchanges: {
-          enabled: false,
-          url: 'http://trackchanges.example.com',
-        },
         project_history: {
           url: this.projectHistoryUrl,
         },
@@ -55,56 +51,45 @@ describe('HistoryManager', function () {
   })
 
   describe('initializeProject', function () {
-    describe('with project history enabled', function () {
-      beforeEach(function () {
-        this.settings.apis.project_history.initializeHistoryForNewProjects = true
-      })
-
-      describe('project history returns a successful response', function () {
-        beforeEach(async function () {
-          this.response.json.resolves({ project: { id: this.historyId } })
-          this.result = await this.HistoryManager.promises.initializeProject(
-            this.historyId
-          )
-        })
+    beforeEach(function () {
+      this.settings.apis.project_history.initializeHistoryForNewProjects = true
+    })
 
-        it('should call the project history api', function () {
-          this.fetch.should.have.been.calledWithMatch(
-            `${this.settings.apis.project_history.url}/project`,
-            { method: 'POST' }
-          )
-        })
+    describe('project history returns a successful response', function () {
+      beforeEach(async function () {
+        this.response.json.resolves({ project: { id: this.historyId } })
+        this.result = await this.HistoryManager.promises.initializeProject(
+          this.historyId
+        )
+      })
 
-        it('should return the overleaf id', function () {
-          expect(this.result).to.equal(this.historyId)
-        })
+      it('should call the project history api', function () {
+        this.fetch.should.have.been.calledWithMatch(
+          `${this.settings.apis.project_history.url}/project`,
+          { method: 'POST' }
+        )
       })
 
-      describe('project history returns a response without the project id', function () {
-        it('should throw an error', async function () {
-          this.response.json.resolves({ project: {} })
-          await expect(
-            this.HistoryManager.promises.initializeProject(this.historyId)
-          ).to.be.rejected
-        })
+      it('should return the overleaf id', function () {
+        expect(this.result).to.equal(this.historyId)
       })
+    })
 
-      describe('project history errors', function () {
-        it('should propagate the error', async function () {
-          this.fetch.rejects(new Error('problem connecting'))
-          await expect(
-            this.HistoryManager.promises.initializeProject(this.historyId)
-          ).to.be.rejected
-        })
+    describe('project history returns a response without the project id', function () {
+      it('should throw an error', async function () {
+        this.response.json.resolves({ project: {} })
+        await expect(
+          this.HistoryManager.promises.initializeProject(this.historyId)
+        ).to.be.rejected
       })
     })
 
-    describe('with project history disabled', function () {
-      it('should return without errors', async function () {
-        this.settings.apis.project_history.initializeHistoryForNewProjects = false
+    describe('project history errors', function () {
+      it('should propagate the error', async function () {
+        this.fetch.rejects(new Error('problem connecting'))
         await expect(
           this.HistoryManager.promises.initializeProject(this.historyId)
-        ).to.be.fulfilled
+        ).to.be.rejected
       })
     })
   })