Overleaf/services/filestore/test/acceptance/js/FilestoreTests.js

const chai = require('chai')
const { expect } = chai
const fs = require('node:fs')
const Stream = require('node:stream')
const Settings = require('@overleaf/settings')
const Path = require('node:path')
const FilestoreApp = require('./FilestoreApp')
const TestHelper = require('./TestHelper')
const fetch = require('node-fetch')
const { promisify } = require('node:util')
const { Storage } = require('@google-cloud/storage')
const streamifier = require('streamifier')
chai.use(require('chai-as-promised'))
const { ObjectId } = require('mongodb')
const ChildProcess = require('node:child_process')

const fsWriteFile = promisify(fs.writeFile)
const fsStat = promisify(fs.stat)
const exec = promisify(ChildProcess.exec)
const msleep = promisify(setTimeout)

if (!process.env.AWS_ACCESS_KEY_ID) {
  throw new Error('please provide credentials for the AWS S3 test server')
}

process.on('unhandledRejection', e => {
  // eslint-disable-next-line no-console
  console.log('** Unhandled Promise Rejection **\n', e)
  throw e
})

// store settings for multiple backends, so that we can test each one.
// fs will always be available - add others if they are configured
const {
  BackendSettings,
  s3Config,
  s3SSECConfig,
  AWS_S3_USER_FILES_STORAGE_CLASS,
} = require('./TestConfig')
const {
  AlreadyWrittenError,
  NotFoundError,
  NotImplementedError,
  NoKEKMatchedError,
} = require('@overleaf/object-persistor/src/Errors')
const {
  PerProjectEncryptedS3Persistor,
  RootKeyEncryptionKey,
} = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor')
const { S3Persistor } = require('@overleaf/object-persistor/src/S3Persistor')
const crypto = require('node:crypto')
const { WritableBuffer } = require('@overleaf/stream-utils')
const { gzipSync } = require('node:zlib')

describe('Filestore', function () {
  this.timeout(1000 * 10)
  const filestoreUrl = `http://127.0.0.1:${Settings.internal.filestore.port}`

  const seenSockets = []
  async function expectNoSockets() {
    try {
      await msleep(1000)
      const { stdout } = await exec('ss -tn')
      const lines = stdout.split('\n')
      const header = lines.shift()

      const badSockets = []
      for (const socket of lines) {
        const fields = socket.split(' ').filter(part => part !== '')
        if (
          fields.length > 2 &&
          parseInt(fields[1]) &&
          !seenSockets.includes(socket)
        ) {
          badSockets.push(socket)
          seenSockets.push(socket)
        }
      }

      if (badSockets.length) {
        // eslint-disable-next-line no-console
        console.error(
          'ERR: Sockets still have receive buffer after connection closed'
        )
        console.error(header)
        for (const socket of badSockets) {
          // eslint-disable-next-line no-console
          console.error(socket)
        }
        throw new Error('Sockets still open after connection closed')
      }
    } catch (err) {
      expect(err).not.to.exist
    }
  }

  // redefine the test suite for every available backend
  for (const [backendVariantWithShardNumber, backendSettings] of Object.entries(
    BackendSettings
  )) {
    describe(backendVariantWithShardNumber, function () {
      let app,
        previousEgress,
        previousIngress,
        metricPrefix,
        projectId,
        otherProjectId

      const dataEncryptionKeySize =
        backendSettings.backend === 's3SSEC' ? 32 : 0

      const BUCKET_NAMES = [
        process.env.GCS_USER_FILES_BUCKET_NAME,
        process.env.GCS_TEMPLATE_FILES_BUCKET_NAME,
        `${process.env.GCS_USER_FILES_BUCKET_NAME}-deleted`,
        `${process.env.GCS_TEMPLATE_FILES_BUCKET_NAME}-deleted`,
      ]

      before('start filestore with new settings', async function () {
        // create the app with the relevant filestore settings
        Settings.filestore = backendSettings
        app = new FilestoreApp()
        await app.runServer()
      })

      if (backendSettings.gcs) {
        before('create gcs buckets', async function () {
          // create test buckets for gcs
          const storage = new Storage(Settings.filestore.gcs.endpoint)
          for (const bucketName of BUCKET_NAMES) {
            await storage.createBucket(bucketName)
          }
        })

        after('delete gcs buckets', async function () {
          // tear down all the gcs buckets
          const storage = new Storage(Settings.filestore.gcs.endpoint)
          for (const bucketName of BUCKET_NAMES) {
            const bucket = storage.bucket(bucketName)
            await bucket.deleteFiles()
            await bucket.delete()
          }
        })
      }

      after('stop filestore app', async function () {
        await app.stop()
      })

      beforeEach('fetch previous egress metric', async function () {
        // retrieve previous metrics from the app
        if (['s3', 's3SSEC', 'gcs'].includes(Settings.filestore.backend)) {
          metricPrefix = Settings.filestore.backend.replace('SSEC', '')
          previousEgress = await TestHelper.getMetric(
            filestoreUrl,
            `${metricPrefix}_egress`
          )
        }
        projectId = new ObjectId().toString()
        otherProjectId = new ObjectId().toString()
      })

      it('should send a 200 for the status endpoint', async function () {
        const response = await fetch(`${filestoreUrl}/status`)
        expect(response.status).to.equal(200)
        const body = await response.text()
        expect(body).to.contain('filestore')
        expect(body).to.contain('up')
      })

      describe('with a file on the server', function () {
        let fileId, fileUrl, constantFileContent

        const localFileReadPath =
          '/tmp/filestore_acceptance_tests_file_read.txt'

        beforeEach('upload file', async function () {
          fileId = new ObjectId().toString()
          fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
          constantFileContent = [
            'hello world',
            `line 2 goes here ${Math.random()}`,
            'there are 3 lines in all',
          ].join('\n')

          await fsWriteFile(localFileReadPath, constantFileContent)

          const readStream = fs.createReadStream(localFileReadPath)
          const res = await fetch(fileUrl, { method: 'POST', body: readStream })
          if (!res.ok) throw new Error(res.statusText)
        })

        beforeEach('retrieve previous ingress metric', async function () {
          // The upload request can bump the ingress metric.
          // The content hash validation might require a full download
          //  in case the ETag field of the upload response is not a md5 sum.
          if (['s3', 's3SSEC', 'gcs'].includes(Settings.filestore.backend)) {
            previousIngress = await TestHelper.getMetric(
              filestoreUrl,
              `${metricPrefix}_ingress`
            )
          }
        })

        it('should return 404 for a non-existant id', async function () {
          const url = fileUrl + '___this_is_clearly_wrong___'
          const response = await fetch(url)
          expect(response.status).to.equal(404)
        })

        it('should return the file size on a HEAD request', async function () {
          const expectedLength = Buffer.byteLength(constantFileContent)
          const res = await fetch(fileUrl, { method: 'HEAD' })
          expect(res.status).to.equal(200)
          expect(res.headers.get('Content-Length')).to.equal(
            expectedLength.toString()
          )
        })

        it('should be able get the file back', async function () {
          const res = await fetch(fileUrl)
          const body = await res.text()
          expect(body).to.equal(constantFileContent)
        })

        it('should send a 200 for the health-check endpoint using the file', async function () {
          Settings.health_check = {
            project_id: projectId,
            file_id: fileId,
          }
          const response = await fetch(`${filestoreUrl}/health_check`)
          expect(response.status).to.equal(200)
          const body = await response.text()
          expect(body).to.equal('OK')
        })

        it('should not leak a socket', async function () {
          const res = await fetch(fileUrl)
          if (!res.ok) throw new Error(res.statusText)
          await res.text()
          await expectNoSockets()
        })

        it('should be able to get back the first 9 bytes of the file', async function () {
          const res = await fetch(fileUrl, { headers: { Range: 'bytes=0-8' } })
          const body = await res.text()
          expect(body).to.equal('hello wor')
        })

        it('should be able to get back bytes 4 through 10 of the file', async function () {
          const res = await fetch(fileUrl, { headers: { Range: 'bytes=4-10' } })
          const body = await res.text()
          expect(body).to.equal('o world')
        })

        it('should be able to delete the file', async function () {
          const response = await fetch(fileUrl, { method: 'DELETE' })
          expect(response.status).to.equal(204)
          const response2 = await fetch(fileUrl)
          expect(response2.status).to.equal(404)
        })

        it('should be able to copy files', async function () {
          const newProjectID = new ObjectId().toString()
          const newFileId = new ObjectId().toString()
          const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}`
          let response = await fetch(newFileUrl, {
            method: 'PUT',
            body: JSON.stringify({
              source: {
                project_id: projectId,
                file_id: fileId,
              },
            }),
            headers: {
              'Content-Type': 'application/json',
            },
          })
          expect(response.status).to.equal(200)
          response = await fetch(fileUrl, { method: 'DELETE' })
          expect(response.status).to.equal(204)
          response = await fetch(newFileUrl)
          const body = await response.text()
          expect(body).to.equal(constantFileContent)
        })

        it('should be able to overwrite the file', async function () {
          const newContent = `here is some different content, ${Math.random()}`
          const readStream = streamifier.createReadStream(newContent)
          await fetch(fileUrl, { method: 'POST', body: readStream })

          const response = await fetch(fileUrl)
          const body = await response.text()
          expect(body).to.equal(newContent)
        })

        describe('IfNoneMatch', function () {
          if (backendSettings.backend === 'fs') {
            it('should refuse to handle IfNoneMatch', async function () {
              await expect(
                app.persistor.sendStream(
                  Settings.filestore.stores.user_files,
                  `${projectId}/${fileId}`,
                  fs.createReadStream(localFileReadPath),
                  { ifNoneMatch: '*' }
                )
              ).to.be.rejectedWith(NotImplementedError)
            })
          } else {
            it('should reject sendStream on the same key with IfNoneMatch', async function () {
              await expect(
                app.persistor.sendStream(
                  Settings.filestore.stores.user_files,
                  `${projectId}/${fileId}`,
                  fs.createReadStream(localFileReadPath),
                  { ifNoneMatch: '*' }
                )
              ).to.be.rejectedWith(AlreadyWrittenError)
            })
            it('should allow sendStream on a different key with IfNoneMatch', async function () {
              await app.persistor.sendStream(
                Settings.filestore.stores.user_files,
                `${projectId}/${fileId}-other`,
                fs.createReadStream(localFileReadPath),
                { ifNoneMatch: '*' }
              )
            })
          }
        })

        if (backendSettings.backend !== 'fs') {
          it('should record an egress metric for the upload', async function () {
            const metric = await TestHelper.getMetric(
              filestoreUrl,
              `${metricPrefix}_egress`
            )
            expect(metric - previousEgress).to.equal(
              constantFileContent.length + dataEncryptionKeySize
            )
          })

          it('should record an ingress metric when downloading the file', async function () {
            const response = await fetch(fileUrl)
            expect(response.ok).to.be.true
            await response.text()
            const metric = await TestHelper.getMetric(
              filestoreUrl,
              `${metricPrefix}_ingress`
            )
            expect(metric - previousIngress).to.equal(
              constantFileContent.length + dataEncryptionKeySize
            )
          })

          it('should record an ingress metric for a partial download', async function () {
            const response = await fetch(fileUrl, {
              headers: { Range: 'bytes=0-8' },
            })
            expect(response.ok).to.be.true
            await response.text()
            const metric = await TestHelper.getMetric(
              filestoreUrl,
              `${metricPrefix}_ingress`
            )
            expect(metric - previousIngress).to.equal(9 + dataEncryptionKeySize)
          })
        }
      })

      describe('with multiple files', function () {
        let fileIds, fileUrls, otherFileUrls, projectUrl, otherProjectUrl
        const localFileReadPaths = [
          '/tmp/filestore_acceptance_tests_file_read_1.txt',
          '/tmp/filestore_acceptance_tests_file_read_2.txt',
          '/tmp/filestore_acceptance_tests_file_read_3.txt',
        ]
        const constantFileContents = [
          [
            'hello world',
            `line 2 goes here ${Math.random()}`,
            'there are 3 lines in all',
          ].join('\n'),
          [
            `for reference: ${Math.random()}`,
            'cats are the best animals',
            'wombats are a close second',
          ].join('\n'),
          [
            `another file: ${Math.random()}`,
            'with multiple lines',
            'the end',
          ].join('\n'),
        ]

        before('create local files', async function () {
          return await Promise.all([
            fsWriteFile(localFileReadPaths[0], constantFileContents[0]),
            fsWriteFile(localFileReadPaths[1], constantFileContents[1]),
            fsWriteFile(localFileReadPaths[2], constantFileContents[2]),
          ])
        })

        beforeEach('upload two files', async function () {
          projectUrl = `${filestoreUrl}/project/${projectId}`
          otherProjectUrl = `${filestoreUrl}/project/${otherProjectId}`
          fileIds = [
            new ObjectId().toString(),
            new ObjectId().toString(),
            new ObjectId().toString(),
          ]
          fileUrls = [
            `${projectUrl}/file/${fileIds[0]}`,
            `${projectUrl}/file/${fileIds[1]}`,
          ]
          otherFileUrls = [`${otherProjectUrl}/file/${fileIds[2]}`]

          await Promise.all([
            fetch(fileUrls[0], {
              method: 'POST',
              body: fs.createReadStream(localFileReadPaths[0]),
            }),
            fetch(fileUrls[1], {
              method: 'POST',
              body: fs.createReadStream(localFileReadPaths[1]),
            }),
            fetch(otherFileUrls[0], {
              method: 'POST',
              body: fs.createReadStream(localFileReadPaths[2]),
            }),
          ])
        })

        it('should get the directory size', async function () {
          const response = await fetch(
            `${filestoreUrl}/project/${projectId}/size`
          )
          const body = await response.text()
          expect(parseInt(JSON.parse(body)['total bytes'])).to.equal(
            constantFileContents[0].length + constantFileContents[1].length
          )
        })

        it('should store the files', async function () {
          for (const index in fileUrls) {
            const response = await fetch(fileUrls[index])
            const body = await response.text()
            expect(body).to.equal(constantFileContents[index])
          }
        })

        it('should be able to delete the project', async function () {
          let response = await fetch(projectUrl, { method: 'DELETE' })
          expect(response.status).to.equal(204)

          for (const index in fileUrls) {
            response = await fetch(fileUrls[index])
            expect(response.status).to.equal(404)
          }
        })

        it('should not delete files in other projects', async function () {
          for (const index in otherFileUrls) {
            const response = await fetch(otherFileUrls[index])
            expect(response.status).to.equal(200)
          }
        })

        it('should not delete a partial project id', async function () {
          const response = await fetch(`${filestoreUrl}/project/5`, {
            method: 'DELETE',
          })
          expect(response.status).to.equal(400)
        })
      })

      describe('with a large file', function () {
        this.timeout(1000 * 20)
        let fileId, fileUrl, largeFileContent, error

        beforeEach('upload large file', async function () {
          fileId = new ObjectId().toString()
          fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`

          largeFileContent = '_wombat_'.repeat(1024 * 1024) // 8 megabytes
          largeFileContent += Math.random()

          const readStream = streamifier.createReadStream(largeFileContent)
          const res = await fetch(fileUrl, { method: 'POST', body: readStream })
          if (!res.ok) throw new Error(res.statusText)
        })

        it('should be able to get the file back', async function () {
          const response = await fetch(fileUrl)
          const body = await response.text()
          expect(body).to.equal(largeFileContent)
        })

        it('should not throw an error', function () {
          expect(error).not.to.exist
        })

        it('should not leak a socket', async function () {
          const response = await fetch(fileUrl)
          await response.text()
          await expectNoSockets()
        })

        it('should not leak a socket if the connection is aborted', async function () {
          const controller = new AbortController()
          const response = await fetch(fileUrl, { signal: controller.signal })
          expect(response.ok).to.be.true
          controller.abort()
          await expectNoSockets()
        })
      })

      if (
        (backendSettings.backend === 's3' && !backendSettings.fallback) ||
        (backendSettings.backend === 'gcs' &&
          backendSettings.fallback?.backend === 's3')
      ) {
        describe('with a file in a specific bucket', function () {
          let constantFileContent, fileId, fileUrl, bucketName

          beforeEach('upload file into random bucket', async function () {
            constantFileContent = `This is a file in a different S3 bucket ${Math.random()}`
            fileId = new ObjectId().toString()
            bucketName = `random-bucket-${new ObjectId().toString()}`
            fileUrl = `${filestoreUrl}/bucket/${bucketName}/key/${fileId}`

            const s3 = new S3Persistor({
              ...s3Config(),
              key: process.env.MINIO_ROOT_USER,
              secret: process.env.MINIO_ROOT_PASSWORD,
            })._getClientForBucket(bucketName)
            await s3
              .createBucket({
                Bucket: bucketName,
              })
              .promise()
            await s3
              .upload({
                Bucket: bucketName,
                Key: fileId,
                Body: constantFileContent,
              })
              .promise()
          })

          it('should get the file from the specified bucket', async function () {
            const response = await fetch(fileUrl)
            const body = await response.text()
            expect(body).to.equal(constantFileContent)
          })
        })
      }

      if (backendSettings.backend === 'gcs') {
        describe('when deleting a file in GCS', function () {
          let fileId, fileUrl, content, error, dateBefore, dateAfter

          beforeEach('upload and delete file', async function () {
            fileId = new ObjectId()
            fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`

            content = '_wombat_' + Math.random()

            const readStream = streamifier.createReadStream(content)
            let res = await fetch(fileUrl, { method: 'POST', body: readStream })
            if (!res.ok) throw new Error(res.statusText)
            dateBefore = new Date()
            res = await fetch(fileUrl, { method: 'DELETE' })
            dateAfter = new Date()
            if (!res.ok) throw new Error(res.statusText)
          })

          it('should not throw an error', function () {
            expect(error).not.to.exist
          })

          it('should copy the file to the deleted-files bucket', async function () {
            let date = dateBefore
            const keys = []
            while (date <= dateAfter) {
              keys.push(`${projectId}/${fileId}-${date.toISOString()}`)
              date = new Date(date.getTime() + 1)
            }
            await TestHelper.expectPersistorToHaveSomeFile(
              app.persistor,
              `${Settings.filestore.stores.user_files}-deleted`,
              keys,
              content
            )
          })

          it('should remove the file from the original bucket', async function () {
            await TestHelper.expectPersistorNotToHaveFile(
              app.persistor,
              Settings.filestore.stores.user_files,
              `${projectId}/${fileId}`
            )
          })
        })
      }

      if (backendSettings.fallback) {
        describe('with a fallback', function () {
          let constantFileContent,
            fileId,
            fileKey,
            fileUrl,
            bucket,
            fallbackBucket

          beforeEach('prepare fallback', function () {
            constantFileContent = `This is yet more file content ${Math.random()}`
            fileId = new ObjectId().toString()
            fileKey = `${projectId}/${fileId}`
            fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`

            bucket = Settings.filestore.stores.user_files
            fallbackBucket = Settings.filestore.fallback.buckets[bucket]
          })

          describe('with a file in the fallback bucket', function () {
            beforeEach('upload into fallback', async function () {
              await TestHelper.uploadStringToPersistor(
                app.persistor.fallbackPersistor,
                fallbackBucket,
                fileKey,
                constantFileContent
              )
            })

            it('should not find file in the primary', async function () {
              await TestHelper.expectPersistorNotToHaveFile(
                app.persistor.primaryPersistor,
                bucket,
                fileKey
              )
            })

            it('should find the file in the fallback', async function () {
              await TestHelper.expectPersistorToHaveFile(
                app.persistor.fallbackPersistor,
                fallbackBucket,
                fileKey,
                constantFileContent
              )
            })

            describe('when copyOnMiss is disabled', function () {
              beforeEach('swap copyOnMiss=false', function () {
                app.persistor.settings.copyOnMiss = false
              })

              it('should fetch the file', async function () {
                const res = await fetch(fileUrl)
                const body = await res.text()
                expect(body).to.equal(constantFileContent)
              })

              it('should not copy the file to the primary', async function () {
                const response = await fetch(fileUrl)
                expect(response.ok).to.be.true
                await response.text()

                await TestHelper.expectPersistorNotToHaveFile(
                  app.persistor.primaryPersistor,
                  bucket,
                  fileKey
                )
              })
            })

            describe('when copyOnMiss is enabled', function () {
              beforeEach('swap copyOnMiss=true', function () {
                app.persistor.settings.copyOnMiss = true
              })

              it('should fetch the file', async function () {
                const res = await fetch(fileUrl)
                const body = await res.text()
                expect(body).to.equal(constantFileContent)
              })

              it('copies the file to the primary', async function () {
                const response = await fetch(fileUrl)
                expect(response.ok).to.be.true
                await response.text()
                // wait for the file to copy in the background
                await msleep(1000)

                await TestHelper.expectPersistorToHaveFile(
                  app.persistor.primaryPersistor,
                  bucket,
                  fileKey,
                  constantFileContent
                )
              })
            })

            describe('when copying a file', function () {
              let newFileId, newFileUrl, newFileKey, opts

              beforeEach('prepare to copy file', function () {
                const newProjectID = new ObjectId().toString()
                newFileId = new ObjectId().toString()
                newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}`
                newFileKey = `${newProjectID}/${newFileId}`

                opts = {
                  method: 'put',
                  body: JSON.stringify({
                    source: {
                      project_id: projectId,
                      file_id: fileId,
                    },
                  }),
                  headers: {
                    'Content-Type': 'application/json',
                  },
                }
              })

              describe('when copyOnMiss is false', function () {
                beforeEach('copy with copyOnMiss=false', async function () {
                  app.persistor.settings.copyOnMiss = false

                  const response = await fetch(newFileUrl, opts)
                  expect(response.status).to.equal(200)
                })

                it('should leave the old file in the old bucket', async function () {
                  await TestHelper.expectPersistorToHaveFile(
                    app.persistor.fallbackPersistor,
                    fallbackBucket,
                    fileKey,
                    constantFileContent
                  )
                })

                it('should not create a new file in the old bucket', async function () {
                  await TestHelper.expectPersistorNotToHaveFile(
                    app.persistor.fallbackPersistor,
                    fallbackBucket,
                    newFileKey
                  )
                })

                it('should create a new file in the new bucket', async function () {
                  await TestHelper.expectPersistorToHaveFile(
                    app.persistor.primaryPersistor,
                    bucket,
                    newFileKey,
                    constantFileContent
                  )
                })

                it('should not copy the old file to the primary with the old key', async function () {
                  // wait for the file to copy in the background
                  await msleep(1000)

                  await TestHelper.expectPersistorNotToHaveFile(
                    app.persistor.primaryPersistor,
                    bucket,
                    fileKey
                  )
                })
              })

              describe('when copyOnMiss is true', function () {
                beforeEach('copy with copyOnMiss=false', async function () {
                  app.persistor.settings.copyOnMiss = true

                  const response = await fetch(newFileUrl, opts)
                  expect(response.status).to.equal(200)
                })

                it('should leave the old file in the old bucket', async function () {
                  await TestHelper.expectPersistorToHaveFile(
                    app.persistor.fallbackPersistor,
                    fallbackBucket,
                    fileKey,
                    constantFileContent
                  )
                })

                it('should not create a new file in the old bucket', async function () {
                  await TestHelper.expectPersistorNotToHaveFile(
                    app.persistor.fallbackPersistor,
                    fallbackBucket,
                    newFileKey
                  )
                })

                it('should create a new file in the new bucket', async function () {
                  await TestHelper.expectPersistorToHaveFile(
                    app.persistor.primaryPersistor,
                    bucket,
                    newFileKey,
                    constantFileContent
                  )
                })

                it('should copy the old file to the primary with the old key', async function () {
                  // wait for the file to copy in the background
                  await msleep(1000)

                  await TestHelper.expectPersistorToHaveFile(
                    app.persistor.primaryPersistor,
                    bucket,
                    fileKey,
                    constantFileContent
                  )
                })
              })
            })
          })

          describe('when sending a file', function () {
            beforeEach('upload file', async function () {
              const readStream =
                streamifier.createReadStream(constantFileContent)
              const res = await fetch(fileUrl, {
                method: 'POST',
                body: readStream,
              })
              if (!res.ok) throw new Error(res.statusText)
            })

            it('should store the file on the primary', async function () {
              await TestHelper.expectPersistorToHaveFile(
                app.persistor.primaryPersistor,
                bucket,
                fileKey,
                constantFileContent
              )
            })

            it('should not store the file on the fallback', async function () {
              await TestHelper.expectPersistorNotToHaveFile(
                app.persistor.fallbackPersistor,
                fallbackBucket,
                `${projectId}/${fileId}`
              )
            })
          })

          describe('when deleting a file', function () {
            describe('when the file exists on the primary', function () {
              beforeEach('upload into primary', async function () {
                await TestHelper.uploadStringToPersistor(
                  app.persistor.primaryPersistor,
                  bucket,
                  fileKey,
                  constantFileContent
                )
              })

              it('should delete the file', async function () {
                const response1 = await fetch(fileUrl, { method: 'DELETE' })
                expect(response1.status).to.equal(204)
                const response2 = await fetch(fileUrl)
                expect(response2.status).to.equal(404)
              })
            })

            describe('when the file exists on the fallback', function () {
              beforeEach('upload into fallback', async function () {
                await TestHelper.uploadStringToPersistor(
                  app.persistor.fallbackPersistor,
                  fallbackBucket,
                  fileKey,
                  constantFileContent
                )
              })

              it('should delete the file', async function () {
                const response1 = await fetch(fileUrl, { method: 'DELETE' })
                expect(response1.status).to.equal(204)
                const response2 = await fetch(fileUrl)
                expect(response2.status).to.equal(404)
              })
            })

            describe('when the file exists on both the primary and the fallback', function () {
              beforeEach(
                'upload into both primary and fallback',
                async function () {
                  await TestHelper.uploadStringToPersistor(
                    app.persistor.primaryPersistor,
                    bucket,
                    fileKey,
                    constantFileContent
                  )
                  await TestHelper.uploadStringToPersistor(
                    app.persistor.fallbackPersistor,
                    fallbackBucket,
                    fileKey,
                    constantFileContent
                  )
                }
              )

              it('should delete the files', async function () {
                const response1 = await fetch(fileUrl, { method: 'DELETE' })
                expect(response1.status).to.equal(204)
                const response2 = await fetch(fileUrl)
                expect(response2.status).to.equal(404)
              })
            })

            describe('when the file does not exist', function () {
              it('should return return 204', async function () {
                // S3 doesn't give us a 404 when the object doesn't exist, so to stay
                // consistent we merrily return 204 ourselves here as well
                const response = await fetch(fileUrl, { method: 'DELETE' })
                expect(response.status).to.equal(204)
              })
            })
          })
        })
      }

      describe('with a pdf file', function () {
        let fileId, fileUrl, localFileSize
        const localFileReadPath = Path.resolve(
          __dirname,
          '../../fixtures/test.pdf'
        )

        beforeEach('upload test.pdf', async function () {
          fileId = new ObjectId().toString()
          fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
          const stat = await fsStat(localFileReadPath)
          localFileSize = stat.size
          const readStream = fs.createReadStream(localFileReadPath)
          const res = await fetch(fileUrl, { method: 'POST', body: readStream })
          if (!res.ok) throw new Error(res.statusText)
        })

        it('should be able get the file back', async function () {
          const response = await fetch(fileUrl)
          const body = await response.text()
          expect(body.substring(0, 8)).to.equal('%PDF-1.5')
        })

        if (backendSettings.backend !== 'fs') {
          it('should record an egress metric for the upload', async function () {
            const metric = await TestHelper.getMetric(
              filestoreUrl,
              `${metricPrefix}_egress`
            )
            expect(metric - previousEgress).to.equal(
              localFileSize + dataEncryptionKeySize
            )
          })
        }

        describe('getting the preview image', function () {
          this.timeout(1000 * 20)
          let previewFileUrl

          beforeEach('prepare previewFileUrl for preview', function () {
            previewFileUrl = `${fileUrl}?style=preview`
          })

          it('should not time out', async function () {
            const response = await fetch(previewFileUrl)
            expect(response.status).to.equal(200)
            await response.arrayBuffer()
          })

          it('should respond with image data', async function () {
            // note: this test relies of the imagemagick conversion working
            const response = await fetch(previewFileUrl)
            expect(response.status).to.equal(200)
            const body = await response.text()
            expect(body.length).to.be.greaterThan(400)
            expect(body.substr(1, 3)).to.equal('PNG')
          })
        })

        describe('warming the cache', function () {
          this.timeout(1000 * 20)
          let previewFileUrl

          beforeEach('prepare previewFileUrl for cacheWarn', function () {
            previewFileUrl = `${fileUrl}?style=preview&cacheWarm=true`
          })

          it('should not time out', async function () {
            const response = await fetch(previewFileUrl)
            expect(response.status).to.equal(200)
            await response.arrayBuffer()
          })

          it('should not leak sockets', async function () {
            const response1 = await fetch(previewFileUrl)
            expect(response1.status).to.equal(200)
            // do not read the response body, should be destroyed immediately
            const response2 = await fetch(previewFileUrl)
            expect(response2.status).to.equal(200)
            // do not read the response body, should be destroyed immediately
            await expectNoSockets()
          })

          it("should respond with only an 'OK'", async function () {
            // note: this test relies of the imagemagick conversion working
            const response = await fetch(previewFileUrl)
            const body = await response.text()
            expect(body).to.equal('OK')
          })
        })
      })

      describe('with server side encryption', function () {
        if (backendSettings.backend !== 's3SSEC') return

        before('sanity check top-level variable', function () {
          expect(dataEncryptionKeySize).to.equal(32)
        })

        let fileId1,
          fileId2,
          fileKey1,
          fileKey2,
          fileKeyOtherProject,
          fileUrl1,
          fileUrl2
        beforeEach('prepare ids', function () {
          fileId1 = new ObjectId().toString()
          fileId2 = new ObjectId().toString()
          fileKey1 = `${projectId}/${fileId1}`
          fileKey2 = `${projectId}/${fileId2}`
          fileKeyOtherProject = `${new ObjectId().toString()}/${new ObjectId().toString()}`
          fileUrl1 = `${filestoreUrl}/project/${projectId}/file/${fileId1}`
          fileUrl2 = `${filestoreUrl}/project/${projectId}/file/${fileId2}`
        })

        beforeEach('ensure DEK is missing', async function () {
          // Cannot use test helper expectPersistorNotToHaveFile here, we need to use the KEK.
          await expect(
            app.persistor.getDataEncryptionKeySize(
              backendSettings.stores.user_files,
              fileKey1
            )
          ).to.rejectedWith(NotFoundError)
        })

        async function createRandomContent(url, suffix = '') {
          const content = Math.random().toString() + suffix
          const res = await fetch(url, {
            method: 'POST',
            body: Stream.Readable.from([content]),
          })
          if (!res.ok) throw new Error(res.statusText)
          return async () => {
            const res = await fetch(url, { method: 'GET' })
            if (!res.ok) throw new Error(res.statusText)
            expect(await res.text()).to.equal(content)
          }
        }

        it('should create a DEK when asked explicitly', async function () {
          await app.persistor.generateDataEncryptionKey(
            backendSettings.stores.user_files,
            fileKey1
          )
          expect(
            await app.persistor.getDataEncryptionKeySize(
              backendSettings.stores.user_files,
              fileKey1
            )
          ).to.equal(32)
        })

        it('should create a DEK from writes', async function () {
          await createRandomContent(fileUrl1)
          expect(
            await app.persistor.getDataEncryptionKeySize(
              backendSettings.stores.user_files,
              fileKey1
            )
          ).to.equal(32)
        })

        it('should not create a DEK from reads', async function () {
          const res = await fetch(fileUrl1, {
            method: 'GET',
          })
          if (res.status !== 404) throw new Error(`${res.status} should be 404`)

          // Cannot use test helper expectPersistorNotToHaveFile here, we need to use the KEK.
          await expect(
            app.persistor.getDataEncryptionKeySize(
              backendSettings.stores.user_files,
              fileKey1
            )
          ).to.rejectedWith(NotFoundError)
        })

        it('should never overwrite a data encryption key', async function () {
          const checkGET = await createRandomContent(fileUrl1)

          await expect(
            app.persistor.generateDataEncryptionKey(
              backendSettings.stores.user_files,
              fileKey1
            )
          ).to.rejectedWith(AlreadyWrittenError)

          await checkGET()
        })

        it('should re-use the data encryption key after a write', async function () {
          const checkGET1 = await createRandomContent(fileUrl1, '1')
          const checkGET2 = await createRandomContent(fileUrl2, '2')
          await checkGET1()
          await checkGET2()
        })

        describe('kek rotation', function () {
          const newKEK = new RootKeyEncryptionKey(
            crypto.generateKeySync('aes', { length: 256 }).export(),
            Buffer.alloc(32)
          )
          const oldKEK = new RootKeyEncryptionKey(
            crypto.generateKeySync('aes', { length: 256 }).export(),
            Buffer.alloc(32)
          )
          const migrationStep0 = new PerProjectEncryptedS3Persistor({
            ...s3SSECConfig(),
            automaticallyRotateDEKEncryption: false,
            async getRootKeyEncryptionKeys() {
              return [oldKEK] // only old key
            },
          })
          const migrationStep1 = new PerProjectEncryptedS3Persistor({
            ...s3SSECConfig(),
            automaticallyRotateDEKEncryption: false,
            async getRootKeyEncryptionKeys() {
              return [oldKEK, newKEK] // new key as fallback
            },
          })
          const migrationStep2 = new PerProjectEncryptedS3Persistor({
            ...s3SSECConfig(),
            automaticallyRotateDEKEncryption: true, // <- different compared to partiallyRotated
            async getRootKeyEncryptionKeys() {
              return [newKEK, oldKEK] // old keys as fallback
            },
          })
          const migrationStep3 = new PerProjectEncryptedS3Persistor({
            ...s3SSECConfig(),
            automaticallyRotateDEKEncryption: true,
            async getRootKeyEncryptionKeys() {
              return [newKEK] // only new key
            },
          })

          async function checkWrites(
            fileKey,
            writer,
            readersSuccess,
            readersFailed
          ) {
            const content = Math.random().toString()
            await writer.sendStream(
              Settings.filestore.stores.user_files,
              fileKey,
              Stream.Readable.from([content])
            )

            for (const persistor of readersSuccess) {
              await TestHelper.expectPersistorToHaveFile(
                persistor,
                backendSettings.stores.user_files,
                fileKey,
                content
              )
            }

            for (const persistor of readersFailed) {
              await expect(
                TestHelper.expectPersistorToHaveFile(
                  persistor,
                  backendSettings.stores.user_files,
                  fileKey,
                  content
                )
              ).to.be.rejectedWith(NoKEKMatchedError)
            }
          }

          const stages = [
            {
              name: 'stage 0 - [old]',
              prev: migrationStep0,
              cur: migrationStep0,
              fail: [migrationStep3],
            },
            {
              name: 'stage 1 - [old,new]',
              prev: migrationStep0,
              cur: migrationStep1,
              fail: [],
            },
            {
              name: 'stage 2 - [new,old]',
              prev: migrationStep1,
              cur: migrationStep2,
              fail: [],
            },
            {
              name: 'stage 3 - [new]',
              prev: migrationStep2,
              cur: migrationStep3,
              fail: [migrationStep0],
            },
          ]

          for (const { name, prev, cur, fail } of stages) {
            describe(name, function () {
              this.timeout(1000 * 30)

              it('can read old writes', async function () {
                await checkWrites(fileKey1, prev, [prev, cur], fail)
                await checkWrites(fileKey2, prev, [prev, cur], fail) // check again after access
                await checkWrites(fileKeyOtherProject, prev, [prev, cur], fail)
              })
              it('can read new writes', async function () {
                await checkWrites(fileKey1, prev, [prev, cur], fail)
                await checkWrites(fileKey2, cur, [prev, cur], fail) // check again after access
                await checkWrites(fileKeyOtherProject, cur, [prev, cur], fail)
              })
            })
          }

          describe('full migration', function () {
            it('can read old writes if rotated in sequence', async function () {
              await checkWrites(
                fileKey1,
                migrationStep0,
                [
                  migrationStep0,
                  migrationStep1,
                  migrationStep2, // migrates
                  migrationStep3,
                ],
                []
              )
            })
            it('cannot read/write if not rotated', async function () {
              await checkWrites(
                fileKey1,
                migrationStep0,
                [migrationStep0],
                [migrationStep3]
              )
            })
          })
        })

        /** @type {import('aws-sdk/clients/s3')} */
        let s3Client
        before('create s3 client', function () {
          s3Client = new S3Persistor(s3Config())._getClientForBucket('')
        })

        async function checkDEKStorage({
          dekBucketKeys = [],
          userFilesBucketKeys = [],
        }) {
          await createRandomContent(fileUrl1)

          const { Contents: dekEntries } = await s3Client
            .listObjectsV2({
              Bucket: process.env.AWS_S3_USER_FILES_DEK_BUCKET_NAME,
              Prefix: `${projectId}/`,
            })
            .promise()
          expect(dekEntries).to.have.length(dekBucketKeys.length)
          // Order is not predictable, use members
          expect(dekEntries.map(o => o.Key)).to.have.members(dekBucketKeys)

          const { Contents: userFilesEntries } = await s3Client
            .listObjectsV2({
              Bucket: backendSettings.stores.user_files,
              Prefix: `${projectId}/`,
            })
            .promise()
          expect(userFilesEntries).to.have.length(userFilesBucketKeys.length)
          // Order is not predictable, use members
          expect(userFilesEntries.map(o => o.Key)).to.have.members(
            userFilesBucketKeys
          )
        }

        it('should use a custom bucket for DEKs', async function () {
          await checkDEKStorage({
            dekBucketKeys: [`${projectId}/dek`],
            userFilesBucketKeys: [fileKey1],
          })
        })

        describe('deleteDirectory', function () {
          let checkGET2
          beforeEach('create files', async function () {
            await createRandomContent(fileUrl1, '1')
            checkGET2 = await createRandomContent(fileUrl2, '2')
          })
          it('should delete sub-folder and keep DEK', async function () {
            await app.persistor.deleteDirectory(
              Settings.filestore.stores.user_files,
              fileKey1 // not really a sub-folder, but it will do for this test.
            )
            expect(
              await app.persistor.checkIfObjectExists(
                Settings.filestore.stores.user_files,
                fileKey1
              )
            ).to.equal(false)
            expect(
              await app.persistor.checkIfObjectExists(
                Settings.filestore.stores.user_files,
                fileKey2
              )
            ).to.equal(true)
            expect(
              await app.persistor.getDataEncryptionKeySize(
                Settings.filestore.stores.user_files,
                fileKey2
              )
            ).to.equal(32)
            await checkGET2()
          })
          it('should delete project folder and DEK', async function () {
            await app.persistor.deleteDirectory(
              Settings.filestore.stores.user_files,
              `${projectId}/`
            )
            expect(
              await app.persistor.checkIfObjectExists(
                Settings.filestore.stores.user_files,
                fileKey1
              )
            ).to.equal(false)
            expect(
              await app.persistor.checkIfObjectExists(
                Settings.filestore.stores.user_files,
                fileKey2
              )
            ).to.equal(false)
            await expect(
              app.persistor.getDataEncryptionKeySize(
                Settings.filestore.stores.user_files,
                fileKey2
              )
            ).to.rejectedWith(NotFoundError)
          })
        })
      })

      describe('getObjectSize', function () {
        it('should return a number', async function () {
          const buf = Buffer.from('hello')
          const fileId = new ObjectId().toString()
          const fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
          const res = await fetch(fileUrl, {
            method: 'POST',
            body: Stream.Readable.from([buf]),
          })
          if (!res.ok) throw new Error(res.statusText)
          expect(
            await app.persistor.getObjectSize(
              Settings.filestore.stores.user_files,
              `${projectId}/${fileId}`
            )
          ).to.equal(buf.byteLength)
        })
      })

      describe('checkIfObjectExists', function () {
        it('should return false when the object does not exist', async function () {
          expect(
            await app.persistor.checkIfObjectExists(
              Settings.filestore.stores.user_files,
              `${projectId}/${new ObjectId().toString()}`
            )
          ).to.equal(false)
        })
        it('should return true when the object exists', async function () {
          const fileId = new ObjectId().toString()
          const fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
          const res = await fetch(fileUrl, {
            method: 'POST',
            body: Stream.Readable.from(['hello']),
          })
          if (!res.ok) throw new Error(res.statusText)
          expect(
            await app.persistor.checkIfObjectExists(
              Settings.filestore.stores.user_files,
              `${projectId}/${fileId}`
            )
          ).to.equal(true)
        })
      })

      if (backendSettings.backend === 's3SSEC') {
        describe('storageClass', function () {
          it('should use the default storage class for dek', async function () {
            const key = `${projectId}/${new ObjectId()}`
            const dekBucket = process.env.AWS_S3_USER_FILES_DEK_BUCKET_NAME
            await app.persistor.sendStream(
              dekBucket,
              key,
              Stream.Readable.from(['hello'])
            )
            expect(
              await app.persistor.getObjectStorageClass(dekBucket, key)
            ).to.equal(undefined)
          })

          it('should use the custom storage class for user files', async function () {
            const key = `${projectId}/${new ObjectId()}`
            await app.persistor.sendStream(
              Settings.filestore.stores.user_files,
              key,
              Stream.Readable.from(['hello'])
            )
            const sc = AWS_S3_USER_FILES_STORAGE_CLASS
            expect(sc).to.exist
            expect(
              await app.persistor.getObjectStorageClass(
                Settings.filestore.stores.user_files,
                key
              )
            ).to.equal(sc)
          })
        })
      }

      describe('autoGunzip', function () {
        let key
        beforeEach('new key', function () {
          key = `${projectId}/${new ObjectId().toString()}`
        })
        this.timeout(60 * 1000)
        const body = Buffer.alloc(10 * 1024 * 1024, 'hello')
        const gzippedBody = gzipSync(body)

        /**
         * @param {string} key
         * @param {Buffer} wantBody
         * @param {boolean} autoGunzip
         * @return {Promise<void>}
         */
        async function checkBodyIsTheSame(key, wantBody, autoGunzip) {
          const s = await app.persistor.getObjectStream(
            Settings.filestore.stores.user_files,
            key,
            { autoGunzip }
          )
          const buf = new WritableBuffer()
          await Stream.promises.pipeline(s, buf)
          expect(buf.getContents()).to.deep.equal(wantBody)
        }

        if (backendSettings.backend === 'fs') {
          it('should refuse to handle autoGunzip', async function () {
            await expect(
              app.persistor.getObjectStream(
                Settings.filestore.stores.user_files,
                key,
                { autoGunzip: true }
              )
            ).to.be.rejectedWith(NotImplementedError)
          })
        } else {
          it('should return the raw body with gzip', async function () {
            await app.persistor.sendStream(
              Settings.filestore.stores.user_files,
              key,
              Stream.Readable.from([gzippedBody]),
              { contentEncoding: 'gzip' }
            )
            expect(
              await app.persistor.getObjectSize(
                Settings.filestore.stores.user_files,
                key
              )
            ).to.equal(gzippedBody.byteLength)
            // raw body with autoGunzip=true
            await checkBodyIsTheSame(key, body, true)
            // gzip body without autoGunzip=false
            await checkBodyIsTheSame(key, gzippedBody, false)
          })
          it('should return the raw body without gzip compression', async function () {
            await app.persistor.sendStream(
              Settings.filestore.stores.user_files,
              key,
              Stream.Readable.from([body])
            )
            expect(
              await app.persistor.getObjectSize(
                Settings.filestore.stores.user_files,
                key
              )
            ).to.equal(body.byteLength)
            // raw body with both autoGunzip options
            await checkBodyIsTheSame(key, body, true)
            await checkBodyIsTheSame(key, body, false)
          })

          it('should return the gzip body without gzip header', async function () {
            await app.persistor.sendStream(
              Settings.filestore.stores.user_files,
              key,
              Stream.Readable.from([gzippedBody])
            )
            expect(
              await app.persistor.getObjectSize(
                Settings.filestore.stores.user_files,
                key
              )
            ).to.equal(gzippedBody.byteLength)
            // gzip body with both autoGunzip options
            await checkBodyIsTheSame(key, gzippedBody, true)
            await checkBodyIsTheSame(key, gzippedBody, false)
          })
        }
      })
    })
  }
})