overleaf/services/document-updater/app/js/PersistenceManager.js

// @ts-check
const { setTimeout } = require('node:timers/promises')
const Settings = require('@overleaf/settings')
const Errors = require('./Errors')
const OError = require('@overleaf/o-error')
const Metrics = require('./Metrics')
const logger = require('@overleaf/logger')
const { fetchJson, RequestFailedError } = require('@overleaf/fetch-utils')

const MAX_ATTEMPTS = 2
const RETRY_DELAY_MS = 10
// We have to be quick with HTTP calls because we're holding a lock that
// expires after 30 seconds. We can't let any errors in the rest of the stack
// hold us up, and need to bail out quickly if there is a problem.
const MAX_HTTP_REQUEST_LENGTH = 5000 // 5 seconds

async function getDocOnce(projectId, docId, options = {}) {
  const timer = new Metrics.Timer('persistenceManager.getDoc')
  const info = { projectId, docId } // for errors
  const url = new URL(
    `/project/${projectId}/doc/${docId}`,
    Settings.apis.web.url
  )
  if (options.peek) {
    // used by resyncs
    url.searchParams.set('peek', 'true')
  }
  const fetchParams = {
    method: 'GET',
    basicAuth: {
      user: Settings.apis.web.user,
      password: Settings.apis.web.pass,
    },
    signal: AbortSignal.timeout(MAX_HTTP_REQUEST_LENGTH),
  }
  try {
    const body = await fetchJson(url, fetchParams)

    if (body.lines == null) {
      throw new Errors.DocumentValidationError(
        'web API response had no doc lines',
        info
      )
    }
    if (body.version == null) {
      throw new Errors.DocumentValidationError(
        'web API response had no valid doc version',
        info
      )
    }
    if (body.pathname == null) {
      throw new Errors.DocumentValidationError(
        'web API response had no valid doc pathname',
        info
      )
    }
    if (!body.pathname) {
      logger.warn(
        { projectId, docId },
        'missing pathname in PersistenceManager getDoc'
      )
      Metrics.inc('pathname', 1, {
        path: 'PersistenceManager.getDoc',
        status: body.pathname === '' ? 'zero-length' : 'undefined',
      })
    }

    if (body.otMigrationStage > 0) {
      // Use history-ot
      body.lines = { content: body.lines.join('\n') }
      body.ranges = {}
    }

    if (!body.projectHistoryId) {
      logger.warn(
        { projectId, docId },
        'projectHistoryId not found for doc from web'
      )
    }
    Metrics.inc('getDoc', 1, { status: '200' })
    return {
      lines: body.lines,
      version: body.version,
      ranges: body.ranges,
      pathname: body.pathname,
      projectHistoryId: body.projectHistoryId?.toString(),
      historyRangesSupport: body.historyRangesSupport || false,
      resolvedCommentIds: body.resolvedCommentIds || [],
    }
  } catch (err) {
    let status
    if (err instanceof RequestFailedError) {
      status = err.response?.status
    } else if (err instanceof Errors.DocumentValidationError) {
      status = 'validation-error'
    } else if (err instanceof Error && 'code' in err) {
      status = err.code
    } else {
      status = 'unknown'
    }
    Metrics.inc('getDoc', 1, { status })
    if (err instanceof RequestFailedError) {
      if (status === 404) {
        throw new Errors.NotFoundError('doc not found', info)
      } else if (status === 413) {
        throw new Errors.FileTooLargeError('doc exceeds maximum size', info)
      } else {
        throw new Errors.WebApiServerError('error accessing web API', {
          ...info,
          status,
        })
      }
    } else if (err instanceof Errors.DocumentValidationError) {
      throw err
    } else {
      throw OError.tag(err, 'getDoc failed', info)
    }
  } finally {
    timer.done()
  }
}

async function setDocOnce(
  projectId,
  docId,
  lines,
  version,
  ranges,
  lastUpdatedAt,
  lastUpdatedBy
) {
  const timer = new Metrics.Timer('persistenceManager.setDoc')

  const info = { projectId, docId } // for errors
  const url = new URL(
    `/project/${projectId}/doc/${docId}`,
    Settings.apis.web.url
  )
  const fetchParams = {
    method: 'POST',
    json: {
      lines,
      ranges,
      version,
      lastUpdatedBy,
      lastUpdatedAt,
    },
    basicAuth: {
      user: Settings.apis.web.user,
      password: Settings.apis.web.pass,
    },
    signal: AbortSignal.timeout(MAX_HTTP_REQUEST_LENGTH),
  }
  try {
    const result = await fetchJson(url, fetchParams)
    Metrics.inc('setDoc', 1, { status: '200' })
    return result
  } catch (err) {
    let status
    if (err instanceof RequestFailedError) {
      status = err.response?.status
    } else if (err instanceof Error && 'code' in err) {
      status = err.code
    } else {
      status = 'unknown'
    }
    Metrics.inc('setDoc', 1, { status })
    if (err instanceof RequestFailedError) {
      if (status === 404) {
        throw new Errors.NotFoundError('doc not found', info)
      } else if (status === 413) {
        throw new Errors.FileTooLargeError('doc exceeds maximum size', info)
      } else {
        throw new Errors.WebApiServerError('error accessing web API', {
          ...info,
          status,
        })
      }
    } else {
      throw OError.tag(err, 'setDoc failed', info)
    }
  } finally {
    timer.done()
  }
}

// Original set of retryable errors from requestretry
const RETRYABLE_ERRORS = new Set([
  'ECONNRESET',
  'ENOTFOUND',
  'ESOCKETTIMEDOUT',
  'ETIMEDOUT',
  'ECONNREFUSED',
  'EHOSTUNREACH',
  'EPIPE',
  'EAI_AGAIN',
  'EBUSY',
])

function isRetryable(error) {
  // use the same retryable errors as requestretry
  // node-fetch uses AbortError:
  // https://github.com/node-fetch/node-fetch/blob/main/docs/ERROR-HANDLING.md
  if (error.name === 'AbortError') {
    return true
  } else if (error instanceof Errors.WebApiServerError) {
    const status = error.info?.status
    return (
      typeof status === 'number' &&
      (status === 429 || (status >= 500 && status < 600))
    )
  } else if (typeof error?.code === 'string') {
    return Boolean(RETRYABLE_ERRORS.has(error.code))
  } else {
    return false
  }
}

async function callWithRetries(name, fn) {
  let remainingAttempts = MAX_ATTEMPTS
  while (true) {
    try {
      const result = await fn()
      if (remainingAttempts < MAX_ATTEMPTS) {
        Metrics.inc(`${name}-retries`, 1, { status: 'success' })
      }
      return result
    } catch (err) {
      remainingAttempts--
      if (remainingAttempts > 0 && isRetryable(err)) {
        await setTimeout(RETRY_DELAY_MS)
        continue
      } else {
        if (remainingAttempts < MAX_ATTEMPTS - 1) {
          Metrics.inc(`${name}-retries`, 1, { status: 'error' })
        }
        throw err
      }
    }
  }
}

async function getDocWithRetries(projectId, docId, options = {}) {
  return await callWithRetries('getDoc', async () => {
    return await getDocOnce(projectId, docId, options)
  })
}

async function setDocWithRetries(
  projectId,
  docId,
  lines,
  version,
  ranges,
  lastUpdatedAt,
  lastUpdatedBy
) {
  return await callWithRetries('setDoc', async () => {
    return await setDocOnce(
      projectId,
      docId,
      lines,
      version,
      ranges,
      lastUpdatedAt,
      lastUpdatedBy
    )
  })
}

module.exports = {
  promises: {
    getDoc: getDocWithRetries,
    setDoc: setDocWithRetries,
  },
}