|
|
@@ -0,0 +1,242 @@
|
|
|
+// @ts-check
|
|
|
+import cleanup from '../storage/support/cleanup.js'
|
|
|
+import fetch from 'node-fetch'
|
|
|
+import testServer from './support/test_backup_server.mjs'
|
|
|
+import { expect } from 'chai'
|
|
|
+import testProjects from './support/test_projects.js'
|
|
|
+import { db } from '../../../../storage/lib/mongodb.js'
|
|
|
+import { ObjectId } from 'mongodb'
|
|
|
+import {
|
|
|
+ backupPersistor,
|
|
|
+ projectBlobsBucket,
|
|
|
+ chunksBucket,
|
|
|
+} from '../../../../storage/lib/backupPersistor.mjs'
|
|
|
+import { makeProjectKey } from '../../../../storage/lib/blob_store/index.js'
|
|
|
+import config from 'config'
|
|
|
+import Stream from 'stream'
|
|
|
+import projectKey from '../../../../storage/lib/project_key.js'
|
|
|
+
|
|
|
+/**
|
|
|
+ * @typedef {import("node-fetch").Response} Response
|
|
|
+ */
|
|
|
+
|
|
|
+const { deksBucket } = config.get('backupStore')
|
|
|
+
|
|
|
+const deletedProjectsCollection = db.collection('deletedProjects')
|
|
|
+
|
|
|
+/**
|
|
|
+ * @param {string} bucket
|
|
|
+ * @param {string} prefix
|
|
|
+ * @return {Promise<Array<string>>}
|
|
|
+ */
|
|
|
+async function listS3Bucket(bucket, prefix) {
|
|
|
+ // @ts-ignore access to internal library helper
|
|
|
+ const client = backupPersistor._getClientForBucket(bucket)
|
|
|
+ const response = await client
|
|
|
+ .listObjectsV2({ Bucket: bucket, Prefix: prefix })
|
|
|
+ .promise()
|
|
|
+ return (response.Contents || []).map(item => item.Key || '')
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * @param {ObjectId} projectId
|
|
|
+ * @return {Promise<Response>}
|
|
|
+ */
|
|
|
+async function deleteProject(projectId) {
|
|
|
+ return await fetch(testServer.testUrl(`/project/${projectId}/backup`), {
|
|
|
+ method: 'DELETE',
|
|
|
+ headers: { Authorization: testServer.basicAuthHeader },
|
|
|
+ })
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * @param {string|ObjectId} historyId
|
|
|
+ * @return {Promise<void>}
|
|
|
+ */
|
|
|
+async function expectToHaveBackup(historyId) {
|
|
|
+ const prefix = projectKey.format(historyId.toString()) + '/'
|
|
|
+ expect(await listS3Bucket(deksBucket, prefix)).to.have.length(1)
|
|
|
+ expect(await listS3Bucket(chunksBucket, prefix)).to.have.length(2)
|
|
|
+ expect(await listS3Bucket(projectBlobsBucket, prefix)).to.have.length(2)
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * @param {string|ObjectId} historyId
|
|
|
+ * @return {Promise<void>}
|
|
|
+ */
|
|
|
+async function expectToHaveNoBackup(historyId) {
|
|
|
+ const prefix = projectKey.format(historyId.toString()) + '/'
|
|
|
+ expect(await listS3Bucket(deksBucket, prefix)).to.have.length(0)
|
|
|
+ expect(await listS3Bucket(chunksBucket, prefix)).to.have.length(0)
|
|
|
+ expect(await listS3Bucket(projectBlobsBucket, prefix)).to.have.length(0)
|
|
|
+}
|
|
|
+
|
|
|
+describe('backupDeletion', function () {
|
|
|
+ beforeEach(cleanup.everything)
|
|
|
+ beforeEach('create health check projects', async function () {
|
|
|
+ await testProjects.createEmptyProject('42')
|
|
|
+ await testProjects.createEmptyProject('000000000000000000000042')
|
|
|
+ })
|
|
|
+ beforeEach(testServer.listenOnRandomPort)
|
|
|
+
|
|
|
+ it('renders 200 on /status', async function () {
|
|
|
+ const response = await fetch(testServer.testUrl('/status'))
|
|
|
+ expect(response.status).to.equal(200)
|
|
|
+ })
|
|
|
+
|
|
|
+ it('renders 200 on /health_check', async function () {
|
|
|
+ const response = await fetch(testServer.testUrl('/health_check'))
|
|
|
+ expect(response.status).to.equal(200)
|
|
|
+ })
|
|
|
+
|
|
|
+ describe('DELETE /project/:projectId', function () {
|
|
|
+ const postgresHistoryId = '1'
|
|
|
+ const projectIdPostgres = new ObjectId('000000000000000000000001')
|
|
|
+ const projectIdMongoDB = new ObjectId('000000000000000000000002')
|
|
|
+ const projectIdNonDeleted = new ObjectId('000000000000000000000003')
|
|
|
+ const projectIdNonExpired = new ObjectId('000000000000000000000004')
|
|
|
+ const projectIdWithChunks = new ObjectId('000000000000000000000005')
|
|
|
+
|
|
|
+ beforeEach('cleanup s3 buckets', async function () {
|
|
|
+ await backupPersistor.deleteDirectory(deksBucket, '')
|
|
|
+ await backupPersistor.deleteDirectory(chunksBucket, '')
|
|
|
+ await backupPersistor.deleteDirectory(projectBlobsBucket, '')
|
|
|
+ })
|
|
|
+
|
|
|
+ beforeEach('populate mongo', async function () {
|
|
|
+ await deletedProjectsCollection.insertMany([
|
|
|
+ {
|
|
|
+ _id: new ObjectId(),
|
|
|
+ project: {
|
|
|
+ _id: projectIdPostgres,
|
|
|
+ overleaf: { history: { id: postgresHistoryId } },
|
|
|
+ },
|
|
|
+ deleterData: {
|
|
|
+ deletedProjectId: projectIdPostgres,
|
|
|
+ deletedAt: new Date('2024-01-01T00:00:00Z'),
|
|
|
+ },
|
|
|
+ },
|
|
|
+ {
|
|
|
+ _id: new ObjectId(),
|
|
|
+ project: {
|
|
|
+ _id: projectIdNonExpired,
|
|
|
+ overleaf: { history: { id: projectIdNonExpired.toString() } },
|
|
|
+ },
|
|
|
+ deleterData: {
|
|
|
+ deletedProjectId: projectIdNonExpired,
|
|
|
+ deletedAt: new Date(),
|
|
|
+ },
|
|
|
+ },
|
|
|
+ ...[projectIdMongoDB, projectIdWithChunks].map(projectId => {
|
|
|
+ return {
|
|
|
+ _id: new ObjectId(),
|
|
|
+ project: {
|
|
|
+ _id: projectId,
|
|
|
+ overleaf: { history: { id: projectId.toString() } },
|
|
|
+ },
|
|
|
+ deleterData: {
|
|
|
+ deletedProjectId: projectId,
|
|
|
+ deletedAt: new Date('2024-01-01T00:00:00Z'),
|
|
|
+ },
|
|
|
+ }
|
|
|
+ }),
|
|
|
+ ])
|
|
|
+ })
|
|
|
+
|
|
|
+ beforeEach('initialize history', async function () {
|
|
|
+ await testProjects.createEmptyProject(projectIdWithChunks.toString())
|
|
|
+ })
|
|
|
+
|
|
|
+ beforeEach('create a file in s3', async function () {
|
|
|
+ const historyIds = [
|
|
|
+ postgresHistoryId,
|
|
|
+ projectIdMongoDB,
|
|
|
+ projectIdNonDeleted,
|
|
|
+ projectIdNonExpired,
|
|
|
+ projectIdWithChunks,
|
|
|
+ ]
|
|
|
+ const jobs = []
|
|
|
+ for (const historyId of historyIds) {
|
|
|
+ jobs.push(
|
|
|
+ backupPersistor.sendStream(
|
|
|
+ projectBlobsBucket,
|
|
|
+ makeProjectKey(historyId, 'a'.repeat(40)),
|
|
|
+ Stream.Readable.from(['blob a']),
|
|
|
+ { contentLength: 6 }
|
|
|
+ )
|
|
|
+ )
|
|
|
+ jobs.push(
|
|
|
+ backupPersistor.sendStream(
|
|
|
+ projectBlobsBucket,
|
|
|
+ makeProjectKey(historyId, 'b'.repeat(40)),
|
|
|
+ Stream.Readable.from(['blob b']),
|
|
|
+ { contentLength: 6 }
|
|
|
+ )
|
|
|
+ )
|
|
|
+ jobs.push(
|
|
|
+ backupPersistor.sendStream(
|
|
|
+ chunksBucket,
|
|
|
+ projectKey.format(historyId) + '/111',
|
|
|
+ Stream.Readable.from(['chunk 1']),
|
|
|
+ { contentLength: 7 }
|
|
|
+ )
|
|
|
+ )
|
|
|
+ jobs.push(
|
|
|
+ backupPersistor.sendStream(
|
|
|
+ chunksBucket,
|
|
|
+ projectKey.format(historyId) + '/222',
|
|
|
+ Stream.Readable.from(['chunk 2']),
|
|
|
+ { contentLength: 7 }
|
|
|
+ )
|
|
|
+ )
|
|
|
+ }
|
|
|
+ await Promise.all(jobs)
|
|
|
+ })
|
|
|
+
|
|
|
+ it('renders 401 without auth', async function () {
|
|
|
+ const response = await fetch(
|
|
|
+ testServer.testUrl('/project/000000000000000000000042/backup'),
|
|
|
+ { method: 'DELETE' }
|
|
|
+ )
|
|
|
+ expect(response.status).to.equal(401)
|
|
|
+ expect(response.headers.get('www-authenticate')).to.match(/^Basic/)
|
|
|
+ })
|
|
|
+
|
|
|
+ it('returns 422 when not deleted', async function () {
|
|
|
+ const response = await deleteProject(projectIdNonDeleted)
|
|
|
+ expect(response.status).to.equal(422)
|
|
|
+ expect(await response.text()).to.equal(
|
|
|
+ 'refusing to delete non-deleted project'
|
|
|
+ )
|
|
|
+ await expectToHaveBackup(projectIdNonDeleted)
|
|
|
+ })
|
|
|
+ it('returns 422 when not expired', async function () {
|
|
|
+ const response = await deleteProject(projectIdNonExpired)
|
|
|
+ expect(response.status).to.equal(422)
|
|
|
+ expect(await response.text()).to.equal(
|
|
|
+ 'refusing to delete non-expired project'
|
|
|
+ )
|
|
|
+ await expectToHaveBackup(projectIdNonExpired)
|
|
|
+ })
|
|
|
+ it('returns 422 when live-history not deleted', async function () {
|
|
|
+ const response = await deleteProject(projectIdWithChunks)
|
|
|
+ expect(response.status).to.equal(422)
|
|
|
+ expect(await response.text()).to.equal(
|
|
|
+ 'refusing to delete project with remaining chunks'
|
|
|
+ )
|
|
|
+ await expectToHaveBackup(projectIdWithChunks)
|
|
|
+ })
|
|
|
+ it('should successfully delete postgres id', async function () {
|
|
|
+ await expectToHaveBackup(postgresHistoryId)
|
|
|
+ const response = await deleteProject(projectIdPostgres)
|
|
|
+ expect(response.status).to.equal(204)
|
|
|
+ await expectToHaveNoBackup(postgresHistoryId)
|
|
|
+ })
|
|
|
+ it('should successfully delete mongo id', async function () {
|
|
|
+ await expectToHaveBackup(projectIdMongoDB)
|
|
|
+ const response = await deleteProject(projectIdMongoDB)
|
|
|
+ expect(response.status).to.equal(204)
|
|
|
+ await expectToHaveNoBackup(projectIdMongoDB)
|
|
|
+ })
|
|
|
+ })
|
|
|
+})
|