Merge pull request #22170 from overleaf/bg-history-v1-copy-blob

add copyBlob support to history-v1

GitOrigin-RevId: 797ea66c37ca938fc906c4dff7bb1c8bf14c031e

services/history-v1/api/controllers/projects.js

@@ -222,6 +222,30 @@ async function getProjectBlob(req, res, next) {
   }
 }
 
+async function copyProjectBlob(req, res, next) {
+  const sourceProjectId = req.swagger.params.copyFrom.value
+  const targetProjectId = req.swagger.params.project_id.value
+  const blobHash = req.swagger.params.hash.value
+  // Check that blob exists in source project
+  const sourceBlobStore = new BlobStore(sourceProjectId)
+  const targetBlobStore = new BlobStore(targetProjectId)
+  const [sourceBlob, targetBlob] = await Promise.all([
+    sourceBlobStore.getBlob(blobHash),
+    targetBlobStore.getBlob(blobHash),
+  ])
+  if (!sourceBlob) {
+    return render.notFound(res)
+  }
+  // Exit early if the blob exists in the target project.
+  // This will also catch global blobs, which always exist.
+  if (targetBlob) {
+    return res.status(HTTPStatus.NO_CONTENT).end()
+  }
+  // Otherwise, copy blob from source project to target project
+  await sourceBlobStore.copyBlob(sourceBlob, targetProjectId)
+  res.status(HTTPStatus.CREATED).end()
+}
+
 async function getSnapshotAtVersion(projectId, version) {
   const chunk = await chunkStore.loadAtVersion(projectId, version)
   const snapshot = chunk.getSnapshot()
@@ -247,4 +271,5 @@ module.exports = {
   deleteProject: expressify(deleteProject),
   createProjectBlob: expressify(createProjectBlob),
   getProjectBlob: expressify(getProjectBlob),
+  copyProjectBlob: expressify(copyProjectBlob),
 }

services/history-v1/api/swagger/projects.js

@@ -134,6 +134,42 @@ exports.paths = {
         },
       },
     },
+    post: {
+      'x-swagger-router-controller': 'projects',
+      operationId: 'copyProjectBlob',
+      tags: ['Project'],
+      description:
+        'Copies a blob from a source project to a target project when duplicating a project',
+      parameters: [
+        {
+          name: 'project_id',
+          in: 'path',
+          description: 'target project id',
+          required: true,
+          type: 'string',
+        },
+        {
+          name: 'hash',
+          in: 'path',
+          description: 'Hexadecimal SHA-1 hash',
+          required: true,
+          type: 'string',
+          pattern: Blob.HEX_HASH_RX_STRING,
+        },
+        {
+          name: 'copyFrom',
+          in: 'query',
+          description: 'source project id',
+          required: false,
+          type: 'string',
+        },
+      ],
+      responses: {
+        201: {
+          description: 'Created',
+        },
+      },
+    },
   },
   '/projects/{project_id}/latest/content': {
     get: {

services/history-v1/storage/lib/blob_store/index.js

@@ -390,6 +390,31 @@ class BlobStore {
     const blob = await this.backend.findBlob(this.projectId, hash)
     return blob
   }
+
+  /**
+   * Copy an existing sourceBlob in this project to a target project.
+   * @param {Blob} sourceBlob
+   * @param {string} targetProjectId
+   * @return {Promise<void>}
+   */
+  async copyBlob(sourceBlob, targetProjectId) {
+    assert.instance(sourceBlob, Blob, 'bad sourceBlob')
+    assert.projectId(targetProjectId, 'bad targetProjectId')
+    const hash = sourceBlob.getHash()
+    const sourceProjectId = this.projectId
+    const { bucket, key: sourceKey } = getBlobLocation(sourceProjectId, hash)
+    const destKey = makeProjectKey(targetProjectId, hash)
+    logger.debug({ sourceProjectId, targetProjectId, hash }, 'copyBlob started')
+    try {
+      await persistor.copyObject(bucket, sourceKey, destKey)
+      await this.backend.insertBlob(targetProjectId, sourceBlob)
+    } finally {
+      logger.debug(
+        { sourceProjectId, targetProjectId, hash },
+        'copyBlob finished'
+      )
+    }
+  }
 }
 
 module.exports = {

services/history-v1/test/acceptance/js/api/project_blobs.test.js

@@ -10,6 +10,11 @@ const testFiles = require('../storage/support/test_files')
 const testServer = require('./support/test_server')
 const { expectHttpError } = require('./support/expect_response')
 
+const { globalBlobs } = require('../../../../storage/lib/mongodb.js')
+const {
+  loadGlobalBlobs,
+} = require('../../../../storage/lib/blob_store/index.js')
+
 describe('Project blobs API', function () {
   const projectId = '123'
 
@@ -119,5 +124,101 @@ describe('Project blobs API', function () {
       )
       expect(response.status).to.equal(HTTPStatus.UNAUTHORIZED)
     })
+
+    it('copies the blob to another project', async function () {
+      const targetProjectId = '456'
+      const targetClient =
+        await testServer.createClientForProject(targetProjectId)
+      const targetToken = testServer.createTokenForProject(targetProjectId)
+      const url = new URL(
+        testServer.url(
+          `/api/projects/${targetProjectId}/blobs/${testFiles.HELLO_TXT_HASH}`
+        )
+      )
+      url.searchParams.append('copyFrom', projectId)
+
+      const response = await fetch(url, {
+        method: 'POST',
+        headers: { Authorization: `Bearer ${targetToken}` },
+      })
+      expect(response.status).to.equal(HTTPStatus.CREATED)
+
+      const newBlobResponse = await targetClient.apis.Project.getProjectBlob({
+        project_id: targetProjectId,
+        hash: testFiles.HELLO_TXT_HASH,
+      })
+      const newBlobResponseText = await newBlobResponse.data.text()
+      expect(newBlobResponseText).to.equal(fileContents.toString())
+    })
+
+    it('skips copying a blob to another project if it already exists', async function () {
+      const targetProjectId = '456'
+      const targetClient =
+        await testServer.createClientForProject(targetProjectId)
+      const targetToken = testServer.createTokenForProject(targetProjectId)
+
+      const fileContents = await fs.promises.readFile(
+        testFiles.path('hello.txt')
+      )
+      const uploadResponse = await fetch(
+        testServer.url(
+          `/api/projects/${targetProjectId}/blobs/${testFiles.HELLO_TXT_HASH}`
+        ),
+        {
+          method: 'PUT',
+          headers: { Authorization: `Bearer ${targetToken}` },
+          body: fileContents,
+        }
+      )
+      expect(uploadResponse.ok).to.be.true
+
+      const url = new URL(
+        testServer.url(
+          `/api/projects/${targetProjectId}/blobs/${testFiles.HELLO_TXT_HASH}`
+        )
+      )
+      url.searchParams.append('copyFrom', projectId)
+
+      const response = await fetch(url, {
+        method: 'POST',
+        headers: { Authorization: `Bearer ${targetToken}` },
+      })
+      expect(response.status).to.equal(HTTPStatus.NO_CONTENT)
+
+      const newBlobResponse = await targetClient.apis.Project.getProjectBlob({
+        project_id: targetProjectId,
+        hash: testFiles.HELLO_TXT_HASH,
+      })
+      const newBlobResponseText = await newBlobResponse.data.text()
+      expect(newBlobResponseText).to.equal(fileContents.toString())
+    })
+  })
+
+  describe('with a global blob', async function () {
+    before(async function () {
+      await globalBlobs.insertOne({
+        _id: testFiles.STRING_A_HASH,
+        byteLength: 1,
+        stringLength: 1,
+      })
+      await loadGlobalBlobs()
+    })
+
+    it('does not copy global blobs', async function () {
+      const targetProjectId = '456'
+      const targetToken = testServer.createTokenForProject(targetProjectId)
+      const url = new URL(
+        testServer.url(
+          `/api/projects/${targetProjectId}/blobs/${testFiles.STRING_A_HASH}`
+        )
+      )
+      url.searchParams.append('copyFrom', projectId)
+
+      const response = await fetch(url, {
+        method: 'POST',
+        headers: { Authorization: `Bearer ${targetToken}` },
+      })
+      expect(response.status).to.equal(HTTPStatus.NO_CONTENT)
+    })
   })
 })

services/history-v1/test/acceptance/js/storage/blob_store.test.js

@@ -478,6 +478,30 @@ describe('BlobStore', function () {
           expect(content).to.equal(globalBlobString)
         })
       })
+
+      describe('copyBlob method', function () {
+        it('copies a binary blob to another project', async function () {
+          const testFile = 'graph.png'
+          const originalHash = testFiles.GRAPH_PNG_HASH
+          const insertedBlob = await blobStore.putFile(testFiles.path(testFile))
+          await blobStore.copyBlob(insertedBlob, scenario.projectId2)
+          const copiedBlob = await blobStore2.getBlob(originalHash)
+          expect(copiedBlob.getHash()).to.equal(originalHash)
+          expect(copiedBlob.getByteLength()).to.equal(
+            insertedBlob.getByteLength()
+          )
+          expect(copiedBlob.getStringLength()).to.be.null
+        })
+
+        it('copies a text blob to another project', async function () {
+          const insertedBlob = await blobStore.putString(helloWorldString)
+          await blobStore.copyBlob(insertedBlob, scenario.projectId2)
+          const copiedBlob = await blobStore2.getBlob(helloWorldHash)
+          expect(copiedBlob.getHash()).to.equal(helloWorldHash)
+          const content = await blobStore2.getString(helloWorldHash)
+          expect(content).to.equal(helloWorldString)
+        })
+      })
     })
   }