Merge pull request #26352 from overleaf/bg-history-redis-add-flush-endpoint

add flush endpoint to history-v1

GitOrigin-RevId: b2ca60f7d040459f9c542e4e87147b9eecc9f596

services/history-v1/api/controllers/project_import.js

@@ -22,6 +22,7 @@ const BlobStore = storage.BlobStore
 const chunkStore = storage.chunkStore
 const HashCheckBlobStore = storage.HashCheckBlobStore
 const commitChanges = storage.commitChanges
+const persistBuffer = storage.persistBuffer
 const InvalidChangeError = storage.InvalidChangeError
 
 const render = require('./render')
@@ -169,5 +170,28 @@ async function importChanges(req, res, next) {
   }
 }
 
+async function flushChanges(req, res, next) {
+  const projectId = req.swagger.params.project_id.value
+  // Use the same limits importChanges, since these are passed to persistChanges
+  const farFuture = new Date()
+  farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
+  const limits = {
+    maxChanges: 0,
+    minChangeTimestamp: farFuture,
+    maxChangeTimestamp: farFuture,
+  }
+  try {
+    await persistBuffer(projectId, limits)
+    res.status(HTTPStatus.OK).end()
+  } catch (err) {
+    if (err instanceof Chunk.NotFoundError) {
+      render.notFound(res)
+    } else {
+      throw err
+    }
+  }
+}
+
 exports.importSnapshot = expressify(importSnapshot)
 exports.importChanges = expressify(importChanges)
+exports.flushChanges = expressify(flushChanges)

services/history-v1/api/swagger/project_import.js

@@ -139,9 +139,45 @@ const getChanges = {
   ],
 }
 
+const flushChanges = {
+  'x-swagger-router-controller': 'project_import',
+  operationId: 'flushChanges',
+  tags: ['ProjectImport'],
+  description: 'Flush project changes from buffer to the chunk store.',
+  parameters: [
+    {
+      name: 'project_id',
+      in: 'path',
+      description: 'project id',
+      required: true,
+      type: 'string',
+    },
+  ],
+  responses: {
+    200: {
+      description: 'Success',
+      schema: {
+        $ref: '#/definitions/Project',
+      },
+    },
+    404: {
+      description: 'Not Found',
+      schema: {
+        $ref: '#/definitions/Error',
+      },
+    },
+  },
+  security: [
+    {
+      basic: [],
+    },
+  ],
+}
+
 exports.paths = {
   '/projects/{project_id}/import': { post: importSnapshot },
   '/projects/{project_id}/legacy_import': { post: importSnapshot },
   '/projects/{project_id}/changes': { get: getChanges, post: importChanges },
   '/projects/{project_id}/legacy_changes': { post: importChanges },
+  '/projects/{project_id}/flush': { post: flushChanges },
 }

services/history-v1/test/acceptance/js/api/project_flush.test.js

@@ -0,0 +1,66 @@
+'use strict'
+
+const BPromise = require('bluebird')
+const { expect } = require('chai')
+const HTTPStatus = require('http-status')
+const fetch = require('node-fetch')
+const fs = BPromise.promisifyAll(require('node:fs'))
+
+const cleanup = require('../storage/support/cleanup')
+const fixtures = require('../storage/support/fixtures')
+const testFiles = require('../storage/support/test_files')
+const testProjects = require('./support/test_projects')
+const testServer = require('./support/test_server')
+
+const { Change, File, Operation } = require('overleaf-editor-core')
+const queueChanges = require('../../../../storage/lib/queue_changes')
+const { getState } = require('../../../../storage/lib/chunk_store/redis')
+
+describe('project flush', function () {
+  beforeEach(cleanup.everything)
+  beforeEach(fixtures.create)
+
+  it('persists queued changes to the chunk store', async function () {
+    const basicAuthClient = testServer.basicAuthClient
+    const projectId = await testProjects.createEmptyProject()
+
+    // upload an empty file
+    const response = await fetch(
+      testServer.url(
+        `/api/projects/${projectId}/blobs/${File.EMPTY_FILE_HASH}`,
+        { qs: { pathname: 'main.tex' } }
+      ),
+      {
+        method: 'PUT',
+        body: fs.createReadStream(testFiles.path('empty.tex')),
+        headers: {
+          Authorization: testServer.basicAuthHeader,
+        },
+      }
+    )
+    expect(response.ok).to.be.true
+
+    const testFile = File.fromHash(File.EMPTY_FILE_HASH)
+    const testChange = new Change(
+      [Operation.addFile('main.tex', testFile)],
+      new Date()
+    )
+    await queueChanges(projectId, [testChange], 0)
+
+    // Verify that the changes are queued and not yet persisted
+    const initialState = await getState(projectId)
+    expect(initialState.persistedVersion).to.be.null
+    expect(initialState.changes).to.have.lengthOf(1)
+
+    const importResponse =
+      await basicAuthClient.apis.ProjectImport.flushChanges({
+        project_id: projectId,
+      })
+
+    expect(importResponse.status).to.equal(HTTPStatus.OK)
+
+    // Verify that the changes were persisted to the chunk store
+    const finalState = await getState(projectId)
+    expect(finalState.persistedVersion).to.equal(1)
+  })
+})