Merge pull request #27025 from overleaf/bg-delete-redis-buffer-when-project-deleted

delete redis buffer when project deleted

GitOrigin-RevId: eef7b6fdeb04cb556ae47794379d83e659f89b2e

services/history-v1/api/controllers/projects.js

@@ -15,11 +15,10 @@ const {
   BlobStore,
   blobHash,
   chunkStore,
+  redisBuffer,
   HashCheckBlobStore,
   ProjectArchive,
   zipStore,
-  persistBuffer,
-  redisBuffer,
 } = require('../../storage')
 
 const render = require('./render')
@@ -229,19 +228,8 @@ async function deleteProject(req, res, next) {
   const projectId = req.swagger.params.project_id.value
   const blobStore = new BlobStore(projectId)
 
-  const farFuture = new Date()
-  farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
-  const limits = {
-    maxChanges: 0,
-    minChangeTimestamp: farFuture,
-    maxChangeTimestamp: farFuture,
-    autoResync: false,
-  }
-
-  await persistBuffer(projectId, limits)
-  await redisBuffer.expireProject(projectId)
-
   await Promise.all([
+    redisBuffer.hardDeleteProject(projectId),
     chunkStore.deleteProjectChunks(projectId),
     blobStore.deleteBlobs(),
   ])

services/history-v1/storage/lib/chunk_store/redis.js

@@ -585,6 +585,53 @@ async function setPersistedVersion(projectId, persistedVersion) {
   }
 }
 
+rclient.defineCommand('hard_delete_project', {
+  numberOfKeys: 6,
+  lua: `
+    local headKey = KEYS[1]
+    local headVersionKey = KEYS[2]
+    local persistedVersionKey = KEYS[3]
+    local expireTimeKey = KEYS[4]
+    local persistTimeKey = KEYS[5]
+    local changesKey = KEYS[6]
+    -- Delete all keys associated with the project
+    redis.call('DEL',
+      headKey,
+      headVersionKey,
+      persistedVersionKey,
+      expireTimeKey,
+      persistTimeKey,
+      changesKey
+    )
+      return 'ok'
+  `,
+})
+
+/** Hard delete a project from Redis by removing all keys associated with it.
+ * This is only to be used when a project is **permanently** deleted.
+ * DO NOT USE THIS FOR ANY OTHER PURPOSES AS IT WILL REMOVE NON-PERSISTED CHANGES.
+ * @param {string} projectId - The unique identifier of the project to delete.
+ * @returns {Promise<string>} A Promise that resolves to 'ok' on success.
+ * @throws {Error} If Redis operations fail.
+ */
+async function hardDeleteProject(projectId) {
+  try {
+    const status = await rclient.hard_delete_project(
+      keySchema.head({ projectId }),
+      keySchema.headVersion({ projectId }),
+      keySchema.persistedVersion({ projectId }),
+      keySchema.expireTime({ projectId }),
+      keySchema.persistTime({ projectId }),
+      keySchema.changes({ projectId })
+    )
+    metrics.inc('chunk_store.redis.hard_delete_project', 1, { status })
+    return status
+  } catch (err) {
+    metrics.inc('chunk_store.redis.hard_delete_project', 1, { status: 'error' })
+    throw err
+  }
+}
+
 rclient.defineCommand('set_expire_time', {
   numberOfKeys: 2,
   lua: `
@@ -794,6 +841,7 @@ module.exports = {
   getChangesSinceVersion,
   getNonPersistedChanges,
   setPersistedVersion,
+  hardDeleteProject,
   setExpireTime,
   expireProject,
   claimExpireJob,

services/history-v1/test/acceptance/js/storage/chunk_store_redis_backend.test.js

@@ -1206,6 +1206,43 @@ describe('chunk buffer Redis backend', function () {
       expect(state.expireTime).to.equal(newTimestamp)
     })
   })
+
+  describe('hardDeleteProject', function () {
+    it('should delete all keys associated with the project', async function () {
+      // Setup project state
+      await setupState(projectId, {
+        headVersion: 5,
+        headSnapshot: new Snapshot(),
+        persistedVersion: 3,
+        persistTime: Date.now(),
+        expireTime: Date.now() + 3600 * 1000, // 1 hour from now
+        changes: 5,
+      })
+
+      // Verify that state exists before deletion
+      let state = await redisBackend.getState(projectId)
+      expect(state.headVersion).to.equal(5)
+
+      // Call hardDeleteProject
+      const result = await redisBackend.hardDeleteProject(projectId)
+      expect(result).to.equal('ok')
+
+      // Verify that all keys are deleted
+      state = await redisBackend.getState(projectId)
+      expect(state.headVersion).to.be.null
+      expect(state.headSnapshot).to.be.null
+      expect(state.persistedVersion).to.be.null
+      expect(state.persistTime).to.be.null
+      expect(state.expireTime).to.be.null
+      expect(state.changes).to.be.an('array').that.is.empty
+    })
+
+    it('should not throw an error if the project does not exist', async function () {
+      // Call hardDeleteProject on a non-existent project
+      const result = await redisBackend.hardDeleteProject(projectId)
+      expect(result).to.equal('ok')
+    })
+  })
 })
 
 async function queueChanges(projectId, changes, opts = {}) {
@@ -1235,6 +1272,7 @@ function makeChange() {
  * @param {string} projectId
  * @param {object} params
  * @param {number} params.headVersion
+ * @param {Snapshot} [params.headSnapshot]
  * @param {number | null} params.persistedVersion
  * @param {number | null} params.persistTime - time when the project should be persisted
  * @param {number | null} params.expireTime - time when the project should expire
@@ -1243,6 +1281,12 @@ function makeChange() {
  */
 async function setupState(projectId, params) {
   await rclient.set(keySchema.headVersion({ projectId }), params.headVersion)
+  if (params.headSnapshot) {
+    await rclient.set(
+      keySchema.head({ projectId }),
+      JSON.stringify(params.headSnapshot.toRaw())
+    )
+  }
   if (params.persistedVersion) {
     await rclient.set(
       keySchema.persistedVersion({ projectId }),