Merge pull request #14029 from overleaf/bg-csh-kustomize-spelling-2

kustomize spelling

GitOrigin-RevId: 5b5203c5de92c3443741cabc081df7afe1063e71

libraries/fetch-utils/.dockerignore

@@ -0,0 +1 @@
+node_modules/

libraries/fetch-utils/.gitignore

@@ -0,0 +1,3 @@
+
+# managed by monorepo$ bin/update_build_scripts
+.npmrc

libraries/fetch-utils/.mocharc.json

@@ -0,0 +1,6 @@
+{
+  "ui": "bdd",
+  "recursive": "true",
+  "reporter": "spec",
+  "require": "test/setup.js"
+}

libraries/fetch-utils/.nvmrc

@@ -0,0 +1 @@
+18.16.1

libraries/fetch-utils/Dockerfile

@@ -0,0 +1,5 @@
+FROM node:18.16.1
+
+WORKDIR /app
+
+USER node

libraries/fetch-utils/buildscript.txt

@@ -0,0 +1,11 @@
+fetch-utils
+--dependencies=None
+--docker-repos=gcr.io/overleaf-ops
+--env-add=
+--env-pass-through=
+--esmock-loader=False
+--has-custom-nodemon=True
+--is-library=True
+--node-version=18.16.1
+--public-repo=False
+--script-version=4.3.0

libraries/fetch-utils/index.js

@@ -0,0 +1,228 @@
+const _ = require('lodash')
+const { Readable } = require('stream')
+const OError = require('@overleaf/o-error')
+const fetch = require('node-fetch')
+
+/**
+ * Make a request and return the parsed JSON response.
+ *
+ * @param {string | URL} url - request URL
+ * @param {object} opts - fetch options
+ * @return {Promise<object>} the parsed JSON response
+ * @throws {RequestFailedError} if the response has a failure status code
+ */
+async function fetchJson(url, opts = {}) {
+  const { json } = await fetchJsonWithResponse(url, opts)
+  return json
+}
+
+async function fetchJsonWithResponse(url, opts = {}) {
+  const { fetchOpts } = parseOpts(opts)
+  fetchOpts.headers = fetchOpts.headers ?? {}
+  fetchOpts.headers.Accept = 'application/json'
+
+  const response = await performRequest(url, fetchOpts)
+  if (!response.ok) {
+    const body = await maybeGetResponseBody(response)
+    throw new RequestFailedError(url, opts, response, body)
+  }
+
+  const json = await response.json()
+  return { json, response }
+}
+
+/**
+ * Make a request and return a stream.
+ *
+ * If the response body is destroyed, the request is aborted.
+ *
+ * @param {string | URL} url - request URL
+ * @param {object} opts - fetch options
+ * @return {Promise<Readable>}
+ * @throws {RequestFailedError} if the response has a failure status code
+ */
+async function fetchStream(url, opts = {}) {
+  const { stream } = await fetchStreamWithResponse(url, opts)
+  return stream
+}
+
+async function fetchStreamWithResponse(url, opts = {}) {
+  const { fetchOpts, abortController } = parseOpts(opts)
+  const response = await performRequest(url, fetchOpts)
+
+  if (!response.ok) {
+    const body = await maybeGetResponseBody(response)
+    throw new RequestFailedError(url, opts, response, body)
+  }
+
+  abortOnDestroyedResponse(abortController, response)
+
+  const stream = response.body
+  return { stream, response }
+}
+
+/**
+ * Make a request and discard the response.
+ *
+ * @param {string | URL} url - request URL
+ * @param {object} opts - fetch options
+ * @return {Promise<Response>}
+ * @throws {RequestFailedError} if the response has a failure status code
+ */
+async function fetchNothing(url, opts = {}) {
+  const { fetchOpts } = parseOpts(opts)
+  const response = await performRequest(url, fetchOpts)
+  if (!response.ok) {
+    const body = await maybeGetResponseBody(response)
+    throw new RequestFailedError(url, opts, response, body)
+  }
+  await discardResponseBody(response)
+  return response
+}
+
+/**
+ * Make a request and return a string.
+ *
+ * @param {string | URL} url - request URL
+ * @param {object} opts - fetch options
+ * @return {Promise<string>}
+ * @throws {RequestFailedError} if the response has a failure status code
+ */
+async function fetchString(url, opts = {}) {
+  const { body } = await fetchStringWithResponse(url, opts)
+  return body
+}
+
+async function fetchStringWithResponse(url, opts = {}) {
+  const { fetchOpts } = parseOpts(opts)
+  const response = await performRequest(url, fetchOpts)
+  if (!response.ok) {
+    const body = await maybeGetResponseBody(response)
+    throw new RequestFailedError(url, opts, response, body)
+  }
+  const body = await response.text()
+  return { body, response }
+}
+
+class RequestFailedError extends OError {
+  constructor(url, opts, response, body) {
+    super('request failed', {
+      url,
+      method: opts.method ?? 'GET',
+      status: response.status,
+    })
+
+    this.response = response
+    if (body != null) {
+      this.body = body
+    }
+  }
+}
+
+function parseOpts(opts) {
+  const fetchOpts = _.omit(opts, ['json', 'signal', 'basicAuth'])
+  if (opts.json) {
+    setupJsonBody(fetchOpts, opts.json)
+  }
+  if (opts.basicAuth) {
+    setupBasicAuth(fetchOpts, opts.basicAuth)
+  }
+
+  const abortController = new AbortController()
+  fetchOpts.signal = abortController.signal
+  if (opts.signal) {
+    abortOnSignal(abortController, opts.signal)
+  }
+  if (opts.body instanceof Readable) {
+    abortOnDestroyedRequest(abortController, fetchOpts.body)
+  }
+  return { fetchOpts, abortController }
+}
+
+function setupJsonBody(fetchOpts, json) {
+  fetchOpts.body = JSON.stringify(json)
+  fetchOpts.headers = fetchOpts.headers ?? {}
+  fetchOpts.headers['Content-Type'] = 'application/json'
+}
+
+function setupBasicAuth(fetchOpts, basicAuth) {
+  fetchOpts.headers = fetchOpts.headers ?? {}
+  fetchOpts.headers.Authorization =
+    'Basic ' +
+    Buffer.from(`${basicAuth.user}:${basicAuth.password}`).toString('base64')
+}
+
+function abortOnSignal(abortController, signal) {
+  const listener = () => {
+    abortController.abort(signal.reason)
+  }
+  if (signal.aborted) {
+    abortController.abort(signal.reason)
+  }
+  signal.addEventListener('abort', listener)
+}
+
+function abortOnDestroyedRequest(abortController, stream) {
+  stream.on('close', () => {
+    if (!stream.readableEnded) {
+      abortController.abort()
+    }
+  })
+}
+
+function abortOnDestroyedResponse(abortController, response) {
+  response.body.on('close', () => {
+    if (!response.bodyUsed) {
+      abortController.abort()
+    }
+  })
+}
+
+async function performRequest(url, fetchOpts) {
+  let response
+  try {
+    response = await fetch(url, fetchOpts)
+  } catch (err) {
+    if (fetchOpts.body instanceof Readable) {
+      fetchOpts.body.destroy()
+    }
+    throw OError.tag(err, err.message, {
+      url,
+      method: fetchOpts.method ?? 'GET',
+    })
+  }
+  if (fetchOpts.body instanceof Readable) {
+    response.body.on('close', () => {
+      if (!fetchOpts.body.readableEnded) {
+        fetchOpts.body.destroy()
+      }
+    })
+  }
+  return response
+}
+
+async function discardResponseBody(response) {
+  // eslint-disable-next-line no-unused-vars
+  for await (const chunk of response.body) {
+    // discard the body
+  }
+}
+
+async function maybeGetResponseBody(response) {
+  try {
+    return await response.text()
+  } catch (err) {
+    return null
+  }
+}
+
+module.exports = {
+  fetchJson,
+  fetchJsonWithResponse,
+  fetchStream,
+  fetchStreamWithResponse,
+  fetchNothing,
+  fetchString,
+  fetchStringWithResponse,
+  RequestFailedError,
+}

libraries/fetch-utils/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@overleaf/fetch-utils",
+  "version": "0.1.0",
+  "description": "utilities for node-fetch",
+  "main": "index.js",
+  "scripts": {
+    "test": "npm run lint && npm run format && npm run test:unit",
+    "test:unit": "mocha",
+    "lint": "eslint --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix .",
+    "format": "prettier --list-different $PWD/'**/*.js'",
+    "format:fix": "prettier --write $PWD/'**/*.js'",
+    "test:ci": "npm run test:unit"
+  },
+  "author": "Overleaf (https://www.overleaf.com)",
+  "license": "AGPL-3.0-only",
+  "devDependencies": {
+    "body-parser": "^1.20.2",
+    "chai": "^4.3.6",
+    "chai-as-promised": "^7.1.1",
+    "express": "^4.18.2",
+    "mocha": "^10.2.0"
+  },
+  "dependencies": {
+    "@overleaf/o-error": "*",
+    "lodash": "^4.17.21",
+    "node-fetch": "^2.6.11"
+  }
+}

libraries/fetch-utils/test/setup.js

@@ -0,0 +1,4 @@
+const chai = require('chai')
+const chaiAsPromised = require('chai-as-promised')
+
+chai.use(chaiAsPromised)

libraries/fetch-utils/test/unit/FetchUtilsTests.js

@@ -0,0 +1,230 @@
+const { expect } = require('chai')
+const { FetchError, AbortError } = require('node-fetch')
+const { Readable } = require('stream')
+const { once } = require('events')
+const { TestServer } = require('./helpers/TestServer')
+const {
+  fetchJson,
+  fetchStream,
+  fetchNothing,
+  fetchString,
+  RequestFailedError,
+} = require('../..')
+
+const PORT = 30001
+
+describe('fetch-utils', function () {
+  before(async function () {
+    this.server = new TestServer()
+    await this.server.start(PORT)
+    this.url = path => `http://localhost:${PORT}${path}`
+  })
+
+  after(async function () {
+    await this.server.stop()
+  })
+
+  describe('fetchJson', function () {
+    it('parses a JSON response', async function () {
+      const json = await fetchJson(this.url('/json/hello'))
+      expect(json).to.deep.equal({ msg: 'hello' })
+    })
+
+    it('parses JSON in the request', async function () {
+      const json = await fetchJson(this.url('/json/add'), {
+        method: 'POST',
+        json: { a: 2, b: 3 },
+      })
+      expect(json).to.deep.equal({ sum: 5 })
+    })
+
+    it('accepts stringified JSON as body', async function () {
+      const json = await fetchJson(this.url('/json/add'), {
+        method: 'POST',
+        body: JSON.stringify({ a: 2, b: 3 }),
+        headers: { 'Content-Type': 'application/json' },
+      })
+      expect(json).to.deep.equal({ sum: 5 })
+    })
+
+    it('throws a FetchError when the payload is not JSON', async function () {
+      await expect(fetchJson(this.url('/hello'))).to.be.rejectedWith(FetchError)
+    })
+
+    it('aborts the request if JSON parsing fails', async function () {
+      await expect(fetchJson(this.url('/large'))).to.be.rejectedWith(FetchError)
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it('handles errors when the payload is JSON', async function () {
+      await expect(fetchJson(this.url('/json/500'))).to.be.rejectedWith(
+        RequestFailedError
+      )
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it('handles errors when the payload is not JSON', async function () {
+      await expect(fetchJson(this.url('/500'))).to.be.rejectedWith(
+        RequestFailedError
+      )
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it('supports abort signals', async function () {
+      await expect(
+        fetchJson(this.url('/hang'), { signal: AbortSignal.timeout(10) })
+      ).to.be.rejectedWith(AbortError)
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it('supports basic auth', async function () {
+      const json = await fetchJson(this.url('/json/basic-auth'), {
+        basicAuth: { user: 'user', password: 'pass' },
+      })
+      expect(json).to.deep.equal({ key: 'verysecret' })
+    })
+
+    it("destroys the request body if it doesn't get consumed", async function () {
+      const stream = Readable.from(infiniteIterator())
+      await fetchJson(this.url('/json/ignore-request'), {
+        method: 'POST',
+        body: stream,
+      })
+      expect(stream.destroyed).to.be.true
+    })
+  })
+
+  describe('fetchStream', function () {
+    it('returns a stream', async function () {
+      const stream = await fetchStream(this.url('/large'))
+      const text = await streamToString(stream)
+      expect(text).to.equal(this.server.largePayload)
+    })
+
+    it('aborts the request when the stream is destroyed', async function () {
+      const stream = await fetchStream(this.url('/large'))
+      stream.destroy()
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it('aborts the request when the request body is destroyed', async function () {
+      const stream = Readable.from(infiniteIterator())
+      const promise = fetchStream(this.url('/hang'), {
+        method: 'POST',
+        body: stream,
+      })
+      stream.destroy()
+      await expect(promise).to.be.rejectedWith(AbortError)
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it('handles errors', async function () {
+      await expect(fetchStream(this.url('/500'))).to.be.rejectedWith(
+        RequestFailedError
+      )
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it('supports abort signals', async function () {
+      await expect(
+        fetchStream(this.url('/hang'), { signal: AbortSignal.timeout(10) })
+      ).to.be.rejectedWith(AbortError)
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it('destroys the request body when an error occurs', async function () {
+      const stream = Readable.from(infiniteIterator())
+      await expect(
+        fetchStream(this.url('/hang'), {
+          body: stream,
+          signal: AbortSignal.timeout(10),
+        })
+      ).to.be.rejectedWith(AbortError)
+      expect(stream.destroyed).to.be.true
+    })
+  })
+
+  describe('fetchNothing', function () {
+    it('closes the connection', async function () {
+      await fetchNothing(this.url('/large'))
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it('aborts the request when the request body is destroyed', async function () {
+      const stream = Readable.from(infiniteIterator())
+      const promise = fetchNothing(this.url('/hang'), {
+        method: 'POST',
+        body: stream,
+      })
+      stream.destroy()
+      await expect(promise).to.be.rejectedWith(AbortError)
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it("doesn't abort the request if the request body ends normally", async function () {
+      const stream = Readable.from('hello there')
+      await fetchNothing(this.url('/sink'), { method: 'POST', body: stream })
+    })
+
+    it('handles errors', async function () {
+      await expect(fetchNothing(this.url('/500'))).to.be.rejectedWith(
+        RequestFailedError
+      )
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it('supports abort signals', async function () {
+      await expect(
+        fetchNothing(this.url('/hang'), { signal: AbortSignal.timeout(10) })
+      ).to.be.rejectedWith(AbortError)
+      await expectRequestAborted(this.server.lastReq)
+    })
+
+    it('destroys the request body when an error occurs', async function () {
+      const stream = Readable.from(infiniteIterator())
+      await expect(
+        fetchNothing(this.url('/hang'), {
+          body: stream,
+          signal: AbortSignal.timeout(10),
+        })
+      ).to.be.rejectedWith(AbortError)
+      expect(stream.destroyed).to.be.true
+    })
+  })
+
+  describe('fetchString', function () {
+    it('returns a string', async function () {
+      const body = await fetchString(this.url('/hello'))
+      expect(body).to.equal('hello')
+    })
+
+    it('handles errors', async function () {
+      await expect(fetchJson(this.url('/500'))).to.be.rejectedWith(
+        RequestFailedError
+      )
+      await expectRequestAborted(this.server.lastReq)
+    })
+  })
+})
+
+async function streamToString(stream) {
+  let s = ''
+  for await (const chunk of stream) {
+    s += chunk
+  }
+  return s
+}
+
+async function* infiniteIterator() {
+  let i = 1
+  while (true) {
+    yield `chunk ${i++}\n`
+  }
+}
+
+async function expectRequestAborted(req) {
+  if (!req.destroyed) {
+    await once(req, 'close')
+    expect(req.destroyed).to.be.true
+  }
+}

libraries/fetch-utils/test/unit/helpers/TestServer.js

@@ -0,0 +1,102 @@
+const express = require('express')
+const bodyParser = require('body-parser')
+
+class TestServer {
+  constructor(port) {
+    this.app = express()
+
+    this.app.use(bodyParser.json())
+    this.app.use((req, res, next) => {
+      this.lastReq = req
+      next()
+    })
+
+    // Plain text endpoints
+
+    this.app.get('/hello', (req, res) => {
+      res.send('hello')
+    })
+
+    this.largePayload = 'x'.repeat(16 * 1024 * 1024)
+    this.app.get('/large', (req, res) => {
+      res.send(this.largePayload)
+    })
+
+    this.app.get('/204', (req, res) => {
+      res.status(204).end()
+    })
+
+    this.app.get('/empty', (req, res) => {
+      res.end()
+    })
+
+    this.app.get('/500', (req, res) => {
+      res.sendStatus(500)
+    })
+
+    this.app.post('/sink', (req, res) => {
+      req.on('data', () => {})
+      req.on('end', () => {
+        res.status(204).end()
+      })
+    })
+
+    // JSON endpoints
+
+    this.app.get('/json/hello', (req, res) => {
+      res.json({ msg: 'hello' })
+    })
+
+    this.app.post('/json/add', (req, res) => {
+      const { a, b } = req.body
+      res.json({ sum: a + b })
+    })
+
+    this.app.get('/json/500', (req, res) => {
+      res.status(500).json({ error: 'Internal server error' })
+    })
+
+    this.app.get('/json/basic-auth', (req, res) => {
+      const expectedAuth = 'Basic ' + Buffer.from('user:pass').toString('base64')
+      if (req.headers.authorization === expectedAuth) {
+        res.json({ key: 'verysecret' })
+      } else {
+        res.status(401).json({ error: 'unauthorized' })
+      }
+    })
+
+    this.app.post('/json/ignore-request', (req, res) => {
+      res.json({ msg: 'hello' })
+    })
+
+    // Never returns
+
+    this.app.post('/hang', (req, res) => {})
+  }
+
+  start(port) {
+    return new Promise((resolve, reject) => {
+      this.server = this.app.listen(port, err => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve()
+        }
+      })
+    })
+  }
+
+  stop() {
+    return new Promise((resolve, reject) => {
+      this.server.close(err => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve()
+        }
+      })
+    })
+  }
+}
+
+module.exports = { TestServer }