Merge pull request #26547 from overleaf/em-revert-expiry-post

Revert "use POST requests for expiring redis buffer from cron"

GitOrigin-RevId: 95e9fd1be7b73699d6fac24035437d467c273d0a

services/history-v1/api/controllers/project_import.js

@@ -28,7 +28,6 @@ const InvalidChangeError = storage.InvalidChangeError
 
 const render = require('./render')
 const Rollout = require('../app/rollout')
-const redisBackend = require('../../storage/lib/chunk_store/redis')
 
 const rollout = new Rollout(config)
 rollout.report(logger) // display the rollout configuration in the logs
@@ -178,13 +177,6 @@ async function flushChanges(req, res, next) {
   }
 }
 
-async function expireProject(req, res, next) {
-  const projectId = req.swagger.params.project_id.value
-  await redisBackend.expireProject(projectId)
-  res.status(HTTPStatus.OK).end()
-}
-
 exports.importSnapshot = expressify(importSnapshot)
 exports.importChanges = expressify(importChanges)
 exports.flushChanges = expressify(flushChanges)
-exports.expireProject = expressify(expireProject)

services/history-v1/api/swagger/project_import.js

@@ -174,46 +174,10 @@ const flushChanges = {
   ],
 }
 
-const expireProject = {
-  'x-swagger-router-controller': 'project_import',
-  operationId: 'expireProject',
-  tags: ['ProjectImport'],
-  description: 'Expire project changes from buffer.',
-  parameters: [
-    {
-      name: 'project_id',
-      in: 'path',
-      description: 'project id',
-      required: true,
-      type: 'string',
-    },
-  ],
-  responses: {
-    200: {
-      description: 'Success',
-      schema: {
-        $ref: '#/definitions/Project',
-      },
-    },
-    404: {
-      description: 'Not Found',
-      schema: {
-        $ref: '#/definitions/Error',
-      },
-    },
-  },
-  security: [
-    {
-      basic: [],
-    },
-  ],
-}
-
 exports.paths = {
   '/projects/{project_id}/import': { post: importSnapshot },
   '/projects/{project_id}/legacy_import': { post: importSnapshot },
   '/projects/{project_id}/changes': { get: getChanges, post: importChanges },
   '/projects/{project_id}/legacy_changes': { post: importChanges },
   '/projects/{project_id}/flush': { post: flushChanges },
-  '/projects/{project_id}/expire': { post: expireProject },
 }

services/history-v1/storage/scripts/expire_redis_chunks.js

@@ -3,47 +3,18 @@ const commandLineArgs = require('command-line-args')
 const redis = require('../lib/redis')
 const { scanAndProcessDueItems } = require('../lib/scan')
 const { expireProject, claimExpireJob } = require('../lib/chunk_store/redis')
-const config = require('config')
-const { fetchNothing } = require('@overleaf/fetch-utils')
 
 const rclient = redis.rclientHistory
 
-const optionDefinitions = [
-  { name: 'dry-run', alias: 'd', type: Boolean },
-  { name: 'post-request', type: Boolean },
-]
+const optionDefinitions = [{ name: 'dry-run', alias: 'd', type: Boolean }]
 const options = commandLineArgs(optionDefinitions)
 const DRY_RUN = options['dry-run'] || false
-const POST_REQUEST = options['post-request'] || false
-const HISTORY_V1_URL = `http://${process.env.HISTORY_V1_HOST || 'localhost'}:${process.env.PORT || 3100}`
 
 logger.initialize('expire-redis-chunks')
 
 async function expireProjectAction(projectId) {
   const job = await claimExpireJob(projectId)
-  if (POST_REQUEST) {
-    await requestProjectExpiry(projectId)
-  } else {
-    await expireProject(projectId)
-  }
-  if (job && job.close) {
-    await job.close()
-  }
-}
-
-async function requestProjectExpiry(projectId) {
-  const job = await claimExpireJob(projectId)
-  logger.debug({ projectId }, 'sending project expire request')
-  const url = `${HISTORY_V1_URL}/api/projects/${projectId}/expire`
-  const credentials = Buffer.from(
-    `staging:${config.get('basicHttpAuth.password')}`
-  ).toString('base64')
-  await fetchNothing(url, {
-    method: 'POST',
-    headers: {
-      Authorization: `Basic ${credentials}`,
-    },
-  })
+  await expireProject(projectId)
   if (job && job.close) {
     await job.close()
   }

services/history-v1/storage/scripts/persist_and_expire_queues.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 node storage/scripts/persist_redis_chunks.mjs --queue --max-time 270
-node storage/scripts/expire_redis_chunks.js --post-request
+node storage/scripts/expire_redis_chunks.js

services/history-v1/test/acceptance/js/api/project_expiry.test.js

@@ -1,82 +0,0 @@
-'use strict'
-
-const BPromise = require('bluebird')
-const { expect } = require('chai')
-const HTTPStatus = require('http-status')
-const fetch = require('node-fetch')
-const fs = BPromise.promisifyAll(require('node:fs'))
-
-const cleanup = require('../storage/support/cleanup')
-const fixtures = require('../storage/support/fixtures')
-const testFiles = require('../storage/support/test_files')
-const testProjects = require('./support/test_projects')
-const testServer = require('./support/test_server')
-
-const { Change, File, Operation } = require('overleaf-editor-core')
-const queueChanges = require('../../../../storage/lib/queue_changes')
-const { getState } = require('../../../../storage/lib/chunk_store/redis')
-
-describe('project expiry', function () {
-  beforeEach(cleanup.everything)
-  beforeEach(fixtures.create)
-
-  it('expire redis buffer', async function () {
-    const basicAuthClient = testServer.basicAuthClient
-    const projectId = await testProjects.createEmptyProject()
-
-    // upload an empty file
-    const response = await fetch(
-      testServer.url(
-        `/api/projects/${projectId}/blobs/${File.EMPTY_FILE_HASH}`,
-        { qs: { pathname: 'main.tex' } }
-      ),
-      {
-        method: 'PUT',
-        body: fs.createReadStream(testFiles.path('empty.tex')),
-        headers: {
-          Authorization: testServer.basicAuthHeader,
-        },
-      }
-    )
-    expect(response.ok).to.be.true
-
-    const testFile = File.fromHash(File.EMPTY_FILE_HASH)
-    const testChange = new Change(
-      [Operation.addFile('main.tex', testFile)],
-      new Date()
-    )
-    await queueChanges(projectId, [testChange], 0)
-
-    // Verify that the changes are queued and not yet persisted
-    const initialState = await getState(projectId)
-    expect(initialState.persistedVersion).to.be.null
-    expect(initialState.changes).to.have.lengthOf(1)
-
-    const importResponse =
-      await basicAuthClient.apis.ProjectImport.flushChanges({
-        project_id: projectId,
-      })
-
-    expect(importResponse.status).to.equal(HTTPStatus.OK)
-
-    // Verify that the changes were persisted to the chunk store
-    const flushedState = await getState(projectId)
-    expect(flushedState.persistedVersion).to.equal(1)
-
-    const expireResponse =
-      await basicAuthClient.apis.ProjectImport.expireProject({
-        project_id: projectId,
-      })
-    expect(expireResponse.status).to.equal(HTTPStatus.OK)
-
-    const finalState = await getState(projectId)
-    expect(finalState).to.deep.equal({
-      changes: [],
-      expireTime: null,
-      headSnapshot: null,
-      headVersion: null,
-      persistTime: null,
-      persistedVersion: null,
-    })
-  })
-})