UrlFetcher.js 2.0 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273
  1. const fs = require('fs')
  2. const logger = require('@overleaf/logger')
  3. const Settings = require('@overleaf/settings')
  4. const {
  5. CustomHttpAgent,
  6. CustomHttpsAgent,
  7. fetchStream,
  8. } = require('@overleaf/fetch-utils')
  9. const { URL } = require('url')
  10. const { pipeline } = require('stream/promises')
  11. const Metrics = require('./Metrics')
  12. const MAX_CONNECT_TIME = 1000
  13. const httpAgent = new CustomHttpAgent({ connectTimeout: MAX_CONNECT_TIME })
  14. const httpsAgent = new CustomHttpsAgent({ connectTimeout: MAX_CONNECT_TIME })
  15. async function pipeUrlToFileWithRetry(url, filePath) {
  16. let remainingAttempts = 3
  17. let lastErr
  18. while (remainingAttempts-- > 0) {
  19. const timer = new Metrics.Timer('url_fetcher', {
  20. path: lastErr ? ' retry' : 'fetch',
  21. })
  22. try {
  23. await pipeUrlToFile(url, filePath)
  24. timer.done({ status: 'success' })
  25. return
  26. } catch (err) {
  27. timer.done({ status: 'error' })
  28. logger.warn(
  29. { err, url, filePath, remainingAttempts },
  30. 'error downloading url'
  31. )
  32. lastErr = err
  33. }
  34. }
  35. throw lastErr
  36. }
  37. async function pipeUrlToFile(url, filePath) {
  38. const u = new URL(url)
  39. if (
  40. Settings.filestoreDomainOveride &&
  41. u.host !== Settings.apis.clsiPerf.host
  42. ) {
  43. url = `${Settings.filestoreDomainOveride}${u.pathname}${u.search}`
  44. }
  45. const stream = await fetchStream(url, {
  46. signal: AbortSignal.timeout(60 * 1000),
  47. // provide a function to get the agent for each request
  48. // as there may be multiple requests with different protocols
  49. // due to redirects.
  50. agent: _url => (_url.protocol === 'https:' ? httpsAgent : httpAgent),
  51. })
  52. const atomicWrite = filePath + '~'
  53. try {
  54. const output = fs.createWriteStream(atomicWrite)
  55. await pipeline(stream, output)
  56. await fs.promises.rename(atomicWrite, filePath)
  57. Metrics.count('UrlFetcher.downloaded_bytes', output.bytesWritten)
  58. } catch (err) {
  59. try {
  60. await fs.promises.unlink(atomicWrite)
  61. } catch (e) {}
  62. throw err
  63. }
  64. }
  65. module.exports.promises = {
  66. pipeUrlToFileWithRetry,
  67. }