Merge pull request #5694 from overleaf/tm-history-scripts-mongo-timeouts

Update v1 without preserveHistory, increase default mongo timeouts, add count query hint

GitOrigin-RevId: bfe1a0024258d0c75646544b9646928b2afe33c1

services/web/scripts/history/count_project_history_categories.js

@@ -4,6 +4,9 @@ const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 5
 const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
 // persist fallback in order to keep batchedUpdate in-sync
 process.env.BATCH_SIZE = BATCH_SIZE
+// raise mongo timeout to 1hr if otherwise unspecified
+process.env.MONGO_SOCKET_TIMEOUT =
+  parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
 
 const { ReadPreference, ObjectId } = require('mongodb')
 const { db } = require('../../app/src/infrastructure/mongodb')
@@ -209,6 +212,9 @@ async function main() {
     _id: 1,
     overleaf: 1,
   }
+  const options = {
+    hint: { _id: 1 },
+  }
   if (VERBOSE_PROJECT_NAMES) {
     projection.name = 1
   }
@@ -216,7 +222,8 @@ async function main() {
     'projects',
     { 'overleaf.history.display': { $ne: true } },
     processBatch,
-    projection
+    projection,
+    options
   )
   console.log('Final')
   console.log(COUNT)

services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js

@@ -5,6 +5,9 @@ const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
 const DRY_RUN = process.env.DRY_RUN !== 'false'
 // persist fallback in order to keep batchedUpdate in-sync
 process.env.BATCH_SIZE = BATCH_SIZE
+// raise mongo timeout to 1hr if otherwise unspecified
+process.env.MONGO_SOCKET_TIMEOUT =
+  parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
 
 const { ReadPreference, ObjectId } = require('mongodb')
 const { db } = require('../../app/src/infrastructure/mongodb')

services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js

@@ -1,10 +1,13 @@
-const SCRIPT_VERSION = 1
+const SCRIPT_VERSION = 2
 const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
 const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
 const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
 const DRY_RUN = process.env.DRY_RUN !== 'false'
 // persist fallback in order to keep batchedUpdate in-sync
 process.env.BATCH_SIZE = BATCH_SIZE
+// raise mongo timeout to 1hr if otherwise unspecified
+process.env.MONGO_SOCKET_TIMEOUT =
+  parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
 
 const { ReadPreference } = require('mongodb')
 const { db } = require('../../app/src/infrastructure/mongodb')
@@ -24,15 +27,23 @@ async function processBatch(_, projects) {
 }
 
 async function processProject(project) {
-  const anyDocHistory = await anyDocHistoryExists(project)
-  if (anyDocHistory) {
-    return
-  }
-  const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-  if (anyDocHistoryIndex) {
-    return
+  const preserveHistory = await shouldPreserveHistory(project)
+  if (preserveHistory) {
+    // if we need to preserve history, then we must bail out if history exists
+    const anyDocHistory = await anyDocHistoryExists(project)
+    if (anyDocHistory) {
+      return
+    }
+    const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
+    if (anyDocHistoryIndex) {
+      return
+    }
+    return await doUpgradeForV1WithoutConversion(project)
+  } else {
+    // if preserveHistory false, then max 7 days of SL history
+    // but v1 already record to both histories, so safe to upgrade
+    return await doUpgradeForV1WithoutConversion(project)
   }
-  await doUpgradeForV1WithoutConversion(project)
 }
 
 async function doUpgradeForV1WithoutConversion(project) {
@@ -54,6 +65,18 @@ async function doUpgradeForV1WithoutConversion(project) {
   RESULT.projectsUpgraded += 1
 }
 
+async function shouldPreserveHistory(project) {
+  return await db.projectHistoryMetaData.findOne(
+    {
+      $and: [
+        { project_id: { $eq: project._id } },
+        { preserveHistory: { $eq: true } },
+      ],
+    },
+    { readPreference: ReadPreference.SECONDARY }
+  )
+}
+
 async function anyDocHistoryExists(project) {
   return await db.docHistory.findOne(
     { project_id: { $eq: project._id } },