Merge pull request #11680 from overleaf/em-mongo-pool-monitoring

Mongo connection pool monitoring

GitOrigin-RevId: 050e50e7e67061ccbf39a710ca4532eafd423365

libraries/metrics/index.js

@@ -251,4 +251,5 @@ module.exports.http = require('./http')
 module.exports.open_sockets = require('./open_sockets')
 module.exports.event_loop = require('./event_loop')
 module.exports.memory = require('./memory')
+module.exports.mongodb = require('./mongodb')
 module.exports.timeAsyncMethod = require('./timeAsyncMethod')

libraries/metrics/mongodb.js

@@ -0,0 +1,54 @@
+const { Gauge } = require('prom-client')
+
+function monitor(mongoClient) {
+  const labelNames = ['mongo_server']
+  const poolSize = new Gauge({
+    name: 'mongo_connection_pool_size',
+    help: 'number of connections in the connection pool',
+    labelNames,
+    // Use this one metric's collect() to set all metrics' values.
+    collect,
+  })
+  const availableConnections = new Gauge({
+    name: 'mongo_connection_pool_available',
+    help: 'number of connections that are not busy',
+    labelNames,
+  })
+  const waitQueueSize = new Gauge({
+    name: 'mongo_connection_pool_waiting',
+    help: 'number of operations waiting for an available connection',
+    labelNames,
+  })
+  const maxPoolSize = new Gauge({
+    name: 'mongo_connection_pool_max',
+    help: 'max size for the connection pool',
+    labelNames,
+  })
+
+  function collect() {
+    // Reset all gauges in case they contain values for servers that
+    // disappeared
+    poolSize.reset()
+    availableConnections.reset()
+    waitQueueSize.reset()
+    maxPoolSize.reset()
+
+    const servers = mongoClient.topology?.s?.servers
+    if (servers != null) {
+      for (const [address, server] of servers) {
+        const pool = server.s?.pool
+        if (pool == null) {
+          continue
+        }
+
+        const labels = { mongo_server: address }
+        poolSize.set(labels, pool.totalConnectionCount)
+        availableConnections.set(labels, pool.availableConnectionCount)
+        waitQueueSize.set(labels, pool.waitQueueSize)
+        maxPoolSize.set(labels, pool.options.maxPoolSize)
+      }
+    }
+  }
+}
+
+module.exports = { monitor }

libraries/metrics/test/unit/js/mongodb.js

@@ -0,0 +1,89 @@
+const Metrics = require('../../..')
+
+const { expect } = require('chai')
+const prom = require('prom-client')
+
+describe('mongodb', function () {
+  beforeEach(function () {
+    prom.register.clear()
+    this.pool = {
+      totalConnectionCount: 8,
+      availableConnectionCount: 2,
+      waitQueueSize: 4,
+      options: { maxPoolSize: 10 },
+    }
+    this.servers = new Map([['server1', { s: { pool: this.pool } }]])
+
+    this.mongoClient = { topology: { s: { servers: this.servers } } }
+  })
+
+  it('handles an unconnected client', async function () {
+    const mongoClient = {}
+    Metrics.mongodb.monitor(mongoClient)
+    const metrics = await getMetrics()
+    expect(metrics).to.deep.equal({})
+  })
+
+  it('collects Mongo metrics', async function () {
+    Metrics.mongodb.monitor(this.mongoClient)
+    const metrics = await getMetrics()
+    expect(metrics).to.deep.equal({
+      'mongo_connection_pool_max:server1': 10,
+      'mongo_connection_pool_size:server1': 8,
+      'mongo_connection_pool_available:server1': 2,
+      'mongo_connection_pool_waiting:server1': 4,
+    })
+  })
+
+  it('handles topology changes', async function () {
+    Metrics.mongodb.monitor(this.mongoClient)
+    let metrics = await getMetrics()
+    expect(metrics).to.deep.equal({
+      'mongo_connection_pool_max:server1': 10,
+      'mongo_connection_pool_size:server1': 8,
+      'mongo_connection_pool_available:server1': 2,
+      'mongo_connection_pool_waiting:server1': 4,
+    })
+
+    // Add a server
+    this.servers.set('server2', this.servers.get('server1'))
+    metrics = await getMetrics()
+    expect(metrics).to.deep.equal({
+      'mongo_connection_pool_max:server1': 10,
+      'mongo_connection_pool_size:server1': 8,
+      'mongo_connection_pool_available:server1': 2,
+      'mongo_connection_pool_waiting:server1': 4,
+      'mongo_connection_pool_max:server2': 10,
+      'mongo_connection_pool_size:server2': 8,
+      'mongo_connection_pool_available:server2': 2,
+      'mongo_connection_pool_waiting:server2': 4,
+    })
+
+    // Delete a server
+    this.servers.delete('server1')
+    metrics = await getMetrics()
+    expect(metrics).to.deep.equal({
+      'mongo_connection_pool_max:server2': 10,
+      'mongo_connection_pool_size:server2': 8,
+      'mongo_connection_pool_available:server2': 2,
+      'mongo_connection_pool_waiting:server2': 4,
+    })
+
+    // Delete another server
+    this.servers.delete('server2')
+    metrics = await getMetrics()
+    expect(metrics).to.deep.equal({})
+  })
+})
+
+async function getMetrics() {
+  const metrics = await prom.register.getMetricsAsJSON()
+  const result = {}
+  for (const metric of metrics) {
+    for (const value of metric.values) {
+      const key = `${metric.name}:${value.labels.mongo_server}`
+      result[key] = value.value
+    }
+  }
+  return result
+}

services/chat/app/js/mongodb.js

@@ -1,3 +1,4 @@
+import Metrics from '@overleaf/metrics'
 import Settings from '@overleaf/settings'
 import { MongoClient } from 'mongodb'
 
@@ -10,3 +11,5 @@ export const db = {
   messages: mongoDb.collection('messages'),
   rooms: mongoDb.collection('rooms'),
 }
+
+Metrics.mongodb.monitor(mongoClient)

services/contacts/app/js/mongodb.js

@@ -1,12 +1,14 @@
+import Metrics from '@overleaf/metrics'
 import Settings from '@overleaf/settings'
 import { MongoClient } from 'mongodb'
 
 export { ObjectId } from 'mongodb'
 
 export const mongoClient = new MongoClient(Settings.mongo.url)
-
 const mongoDb = mongoClient.db()
 
 export const db = {
   contacts: mongoDb.collection('contacts'),
 }
+
+Metrics.mongodb.monitor(mongoClient)

services/docstore/app/js/mongodb.js

@@ -1,3 +1,4 @@
+const Metrics = require('@overleaf/metrics')
 const Settings = require('@overleaf/settings')
 const { MongoClient, ObjectId } = require('mongodb')
 
@@ -9,6 +10,8 @@ const db = {
   docOps: mongoDb.collection('docOps'),
 }
 
+Metrics.mongodb.monitor(mongoClient)
+
 module.exports = {
   db,
   mongoClient,

services/document-updater/app/js/mongodb.js

@@ -1,3 +1,4 @@
+const Metrics = require('@overleaf/metrics')
 const Settings = require('@overleaf/settings')
 const { MongoClient, ObjectId } = require('mongodb')
 
@@ -17,6 +18,8 @@ async function healthCheck() {
   }
 }
 
+Metrics.mongodb.monitor(mongoClient)
+
 module.exports = {
   db,
   ObjectId,

services/history-v1/storage/lib/mongodb.js

@@ -1,3 +1,5 @@
+const Metrics = require('@overleaf/metrics')
+
 const config = require('config')
 const { MongoClient } = require('mongodb')
 
@@ -9,4 +11,6 @@ const blobs = db.collection('projectHistoryBlobs')
 const globalBlobs = db.collection('projectHistoryGlobalBlobs')
 const shardedBlobs = db.collection('projectHistoryShardedBlobs')
 
+Metrics.mongodb.monitor(client)
+
 module.exports = { client, db, chunks, blobs, globalBlobs, shardedBlobs }

services/notifications/app/js/mongodb.js

@@ -1,3 +1,4 @@
+const Metrics = require('@overleaf/metrics')
 const Settings = require('@overleaf/settings')
 const { MongoClient, ObjectId } = require('mongodb')
 
@@ -8,6 +9,8 @@ const db = {
   notifications: mongoDb.collection('notifications'),
 }
 
+Metrics.mongodb.monitor(mongoClient)
+
 module.exports = {
   db,
   mongoClient,

services/project-history/app/js/mongodb.js

@@ -1,3 +1,4 @@
+import Metrics from '@overleaf/metrics'
 import Settings from '@overleaf/settings'
 import { MongoClient } from 'mongodb'
 
@@ -6,6 +7,8 @@ export { ObjectId } from 'mongodb'
 export const mongoClient = new MongoClient(Settings.mongo.url)
 const mongoDb = mongoClient.db()
 
+Metrics.mongodb.monitor(mongoClient)
+
 export const db = {
   deletedProjects: mongoDb.collection('deletedProjects'),
   projects: mongoDb.collection('projects'),

services/web/app/src/infrastructure/mongodb.js

@@ -1,9 +1,6 @@
 const Metrics = require('@overleaf/metrics')
 const { ObjectId } = require('mongodb')
 const OError = require('@overleaf/o-error')
-const {
-  addOptionalCleanupHandlerAfterDrainingConnections,
-} = require('./GracefulShutdown')
 const { getNativeDb } = require('./Mongoose')
 
 if (
@@ -26,7 +23,7 @@ async function waitForDb() {
 const db = {}
 async function setupDb() {
   const internalDb = await getNativeDb()
-  collectStatsForDb(internalDb, 'shared')
+  Metrics.mongodb.monitor(internalDb)
 
   db.contacts = internalDb.collection('contacts')
   db.deletedFiles = internalDb.collection('deletedFiles')
@@ -110,34 +107,6 @@ async function getCollectionInternal(name) {
   return internalDb.collection(name)
 }
 
-function collectStatsForDb(internalDb, label) {
-  const collectOnce = () => {
-    for (const [name, server] of internalDb.s.topology.s.servers.entries()) {
-      const { availableConnectionCount, waitQueueSize } = server.s.pool
-      const { maxPoolSize } = server.s.pool.options
-      const opts = { status: `${label}:${name}` }
-      Metrics.gauge('mongo_connection_pool_max', maxPoolSize, 1, opts)
-      Metrics.gauge(
-        'mongo_connection_pool_available',
-        availableConnectionCount,
-        1,
-        opts
-      )
-      Metrics.gauge('mongo_connection_pool_waiting', waitQueueSize, 1, opts)
-    }
-  }
-  collectOnce() // init metrics
-  const intervalHandle = setInterval(collectOnce, 60_000)
-  addOptionalCleanupHandlerAfterDrainingConnections(
-    `collect mongo connection pool metrics (${label})`,
-    () => {
-      clearInterval(intervalHandle)
-      // collect one more time.
-      collectOnce()
-    }
-  )
-}
-
 module.exports = {
   db,
   ObjectId,
@@ -145,5 +114,4 @@ module.exports = {
   getCollectionInternal,
   dropTestDatabase,
   waitForDb,
-  collectStatsForDb,
 }