[web] display project history blob size on admin pages (#28351)

* [history-v1] add endpoint for getting blob stats of projects

* [web] display project history blob size on admin pages

* [web] break down history storage size by text and binary blob

GitOrigin-RevId: bfa4d56cf2c503d03005c13a0f7ac38158156cd2

services/history-v1/api/controllers/projects.js

@@ -25,6 +25,8 @@ const render = require('./render')
 const expressify = require('./expressify')
 const withTmpDir = require('./with_tmp_dir')
 const StreamSizeLimit = require('./stream_size_limit')
+const { getProjectBlobsBatch } = require('../../storage/lib/blob_store')
+const assert = require('../../storage/lib/assert')
 
 const pipeline = promisify(Stream.pipeline)
 
@@ -378,6 +380,40 @@ async function getSnapshotAtVersion(projectId, version) {
   return snapshot
 }
 
+function sumUpByteLength(blobs) {
+  return blobs.reduce((sum, blob) => sum + blob.getByteLength(), 0)
+}
+
+async function getProjectBlobsStats(req, res) {
+  const projectIds = req.swagger.params.body.value.projectIds
+  const { blobs } = await getProjectBlobsBatch(
+    projectIds.map(id => {
+      if (assert.POSTGRES_ID_REGEXP.test(id)) {
+        return parseInt(id, 10)
+      } else {
+        return id
+      }
+    })
+  )
+  const sizes = []
+  for (const projectId of projectIds) {
+    const projectBlobs = blobs.get(projectId) || []
+    const textBlobs = projectBlobs.filter(b => b.getStringLength() !== null)
+    const binaryBlobs = projectBlobs.filter(b => b.getStringLength() === null)
+    const textBlobBytes = sumUpByteLength(textBlobs)
+    const binaryBlobBytes = sumUpByteLength(binaryBlobs)
+    sizes.push({
+      projectId,
+      textBlobBytes,
+      binaryBlobBytes,
+      totalBytes: textBlobBytes + binaryBlobBytes,
+      nTextBlobs: textBlobs.length,
+      nBinaryBlobs: binaryBlobs.length,
+    })
+  }
+  res.json(sizes)
+}
+
 module.exports = {
   initializeProject: expressify(initializeProject),
   getLatestContent: expressify(getLatestContent),
@@ -396,4 +432,5 @@ module.exports = {
   getProjectBlob: expressify(getProjectBlob),
   headProjectBlob: expressify(headProjectBlob),
   copyProjectBlob: expressify(copyProjectBlob),
+  getProjectBlobsStats: expressify(getProjectBlobsStats),
 }

services/history-v1/api/swagger/index.js

@@ -32,6 +32,28 @@ module.exports = {
       },
       required: ['projectId'],
     },
+    ProjectBlobStats: {
+      properties: {
+        projectId: {
+          type: 'string',
+        },
+        textBlobBytes: {
+          type: 'integer',
+        },
+        binaryBlobBytes: {
+          type: 'integer',
+        },
+        totalBytes: {
+          type: 'integer',
+        },
+        nTextBlobs: {
+          type: 'integer',
+        },
+        nBinaryBlobs: {
+          type: 'integer',
+        },
+      },
+    },
     File: {
       properties: {
         hash: {

services/history-v1/api/swagger/projects.js

@@ -37,6 +37,46 @@ exports.paths = {
       ],
     },
   },
+  '/projects/blob-stats': {
+    post: {
+      'x-swagger-router-controller': 'projects',
+      operationId: 'getProjectBlobsStats',
+      tags: ['Project'],
+      description: 'Get Blob stats for projects.',
+      consumes: ['application/json'],
+      parameters: [
+        {
+          name: 'body',
+          in: 'body',
+          schema: {
+            type: 'object',
+            properties: {
+              projectIds: {
+                type: 'array',
+                items: { type: 'string' },
+              },
+            },
+          },
+        },
+      ],
+      responses: {
+        200: {
+          description: 'Initialized',
+          schema: {
+            type: 'array',
+            items: {
+              $ref: '#/definitions/ProjectBlobStats',
+            },
+          },
+        },
+      },
+      security: [
+        {
+          basic: [],
+        },
+      ],
+    },
+  },
   '/projects/{project_id}': {
     delete: {
       'x-swagger-router-controller': 'projects',

services/history-v1/test/acceptance/js/api/projects.test.js

@@ -30,6 +30,7 @@ const {
   TextOperation,
 } = require('overleaf-editor-core')
 const testProjects = require('./support/test_projects')
+const { ObjectId } = require('mongodb')
 
 describe('project controller', function () {
   beforeEach(cleanup.everything)
@@ -104,6 +105,165 @@ describe('project controller', function () {
     })
   })
 
+  describe('blob stats', function () {
+    let populatedPostgresProjectId,
+      populatedMongoProjectId,
+      emptyPostgresProjectId,
+      emptyMongoProjectId
+
+    async function populateProject(projectId) {
+      const files = {
+        [testFiles.GRAPH_PNG_HASH]: testFiles.path('graph.png'),
+        [testFiles.HELLO_TXT_HASH]: testFiles.path('hello.txt'),
+      }
+      for (const [hash, path] of Object.entries(files)) {
+        const response = await fetch(
+          testServer.url(`/api/projects/${projectId}/blobs/${hash}`),
+          {
+            method: 'PUT',
+            body: fs.createReadStream(path),
+            headers: {
+              Authorization: testServer.basicAuthHeader,
+            },
+          }
+        )
+        expect(response.status).to.equal(201)
+      }
+    }
+
+    beforeEach(async function () {
+      emptyPostgresProjectId = await testProjects.createEmptyProject()
+      emptyMongoProjectId = await testProjects.createEmptyProject(
+        new ObjectId().toString()
+      )
+
+      populatedPostgresProjectId = await testProjects.createEmptyProject()
+      await populateProject(populatedPostgresProjectId)
+      populatedMongoProjectId = await testProjects.createEmptyProject(
+        new ObjectId().toString()
+      )
+      await populateProject(populatedMongoProjectId)
+    })
+
+    it('handles empty postgres project', async function () {
+      const { body } =
+        await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({
+          body: { projectIds: [emptyPostgresProjectId] },
+        })
+      expect(body).to.deep.equal([
+        {
+          projectId: emptyPostgresProjectId,
+          textBlobBytes: 0,
+          binaryBlobBytes: 0,
+          totalBytes: 0,
+          nTextBlobs: 0,
+          nBinaryBlobs: 0,
+        },
+      ])
+    })
+    it('handles populated postgres project', async function () {
+      const { body } =
+        await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({
+          body: { projectIds: [populatedPostgresProjectId] },
+        })
+      expect(body).to.deep.equal([
+        {
+          projectId: populatedPostgresProjectId,
+          textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH,
+          binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH,
+          totalBytes:
+            testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH,
+          nTextBlobs: 1,
+          nBinaryBlobs: 1,
+        },
+      ])
+    })
+
+    it('handles empty mongo project', async function () {
+      const { body } =
+        await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({
+          body: { projectIds: [emptyMongoProjectId] },
+        })
+      expect(body).to.deep.equal([
+        {
+          projectId: emptyMongoProjectId,
+          textBlobBytes: 0,
+          binaryBlobBytes: 0,
+          totalBytes: 0,
+          nTextBlobs: 0,
+          nBinaryBlobs: 0,
+        },
+      ])
+    })
+    it('handles populated mongo project', async function () {
+      const { body } =
+        await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({
+          body: { projectIds: [populatedMongoProjectId] },
+        })
+      expect(body).to.deep.equal([
+        {
+          projectId: populatedMongoProjectId,
+          textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH,
+          binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH,
+          totalBytes:
+            testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH,
+          nTextBlobs: 1,
+          nBinaryBlobs: 1,
+        },
+      ])
+    })
+
+    it('handles batch of projects', async function () {
+      const { body } =
+        await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({
+          body: {
+            projectIds: [
+              populatedPostgresProjectId,
+              populatedMongoProjectId,
+              emptyPostgresProjectId,
+              emptyMongoProjectId,
+            ],
+          },
+        })
+      expect(body).to.deep.equal([
+        {
+          projectId: populatedPostgresProjectId,
+          textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH,
+          binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH,
+          totalBytes:
+            testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH,
+          nTextBlobs: 1,
+          nBinaryBlobs: 1,
+        },
+        {
+          projectId: populatedMongoProjectId,
+          textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH,
+          binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH,
+          totalBytes:
+            testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH,
+          nTextBlobs: 1,
+          nBinaryBlobs: 1,
+        },
+        {
+          projectId: emptyPostgresProjectId,
+          textBlobBytes: 0,
+          binaryBlobBytes: 0,
+          totalBytes: 0,
+          nTextBlobs: 0,
+          nBinaryBlobs: 0,
+        },
+        {
+          projectId: emptyMongoProjectId,
+          textBlobBytes: 0,
+          binaryBlobBytes: 0,
+          totalBytes: 0,
+          nTextBlobs: 0,
+          nBinaryBlobs: 0,
+        },
+      ])
+    })
+  })
+
   // eslint-disable-next-line mocha/no-skipped-tests
   describe.skip('getLatestContent', function () {
     // TODO: remove this endpoint entirely, see

services/web/app/src/Features/History/HistoryManager.js

@@ -307,6 +307,14 @@ async function getHistoryId(projectId) {
   return historyId
 }
 
+async function getProjectBlobStats(historyIds) {
+  return await fetchJson(`${HISTORY_V1_URL}/projects/blob-stats`, {
+    method: 'POST',
+    basicAuth: HISTORY_V1_BASIC_AUTH,
+    json: { projectIds: historyIds.map(id => id.toString()) },
+  })
+}
+
 async function injectUserDetails(data) {
   // data can be either:
   // {
@@ -418,5 +426,6 @@ module.exports = {
     requestBlobWithProjectId,
     getLatestHistory,
     getChanges,
+    getProjectBlobStats,
   },
 }

services/web/frontend/extracted-translations.json

@@ -1612,6 +1612,7 @@
   "sign_up": "",
   "simple_search_mode": "",
   "single_sign_on_sso": "",
+  "size": "",
   "something_not_right": "",
   "something_went_wrong": "",
   "something_went_wrong_canceling_your_subscription": "",

services/web/frontend/js/utils/meta.ts

@@ -151,6 +151,15 @@ export interface Meta {
   'ol-hasTrackChangesFeature': boolean
   'ol-hasWriteAccess': boolean
   'ol-hideLinkingWidgets': boolean // CI only
+  'ol-historyBlobStats': {
+    projectId: string
+    textBlobsBytes: number
+    binaryBlobsBytes: number
+    totalBytes: number
+    nTextBlobs: number
+    nBinaryBlobs: number
+    owned?: boolean
+  }
   'ol-i18n': { currentLangCode: string }
   'ol-inactiveTutorials': string[]
   'ol-institutionEmailNonCanonical': string | undefined

services/web/locales/en.json

@@ -2089,6 +2089,7 @@
   "site_wide_option_available": "Site-wide option available",
   "six_collaborators_per_project": "6 collaborators per project",
   "six_per_project": "6 per project",
+  "size": "Size",
   "skip": "Skip",
   "skip_to_content": "Skip to content",
   "solutions": "Solutions",

services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs

@@ -16,6 +16,21 @@ class MockV1HistoryApi extends AbstractMockApi {
   }
 
   applyRoutes() {
+    this.app.post('/api/projects/blob-stats', (req, res, next) => {
+      res.json(
+        req.body.projectIds.map(projectId => {
+          return {
+            projectId,
+            textBlobBytes: 7331,
+            binaryBlobBytes: 1337,
+            totalBytes: 7331 + 1337,
+            nTextBlobs: 13,
+            nBinaryBlobs: 42,
+          }
+        })
+      )
+    })
+
     this.app.get(
       '/api/projects/:project_id/version/:version/zip',
       (req, res, next) => {