MigrationPersistor.js 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228
  1. const metrics = require('metrics-sharelatex')
  2. const Settings = require('settings-sharelatex')
  3. const logger = require('logger-sharelatex')
  4. const Stream = require('stream')
  5. const { callbackify, promisify } = require('util')
  6. const { NotFoundError, WriteError } = require('./Errors')
  7. const pipeline = promisify(Stream.pipeline)
  8. // Persistor that wraps two other persistors. Talks to the 'primary' by default,
  9. // but will fall back to an older persistor in the case of a not-found error.
  10. // If `Settings.filestore.fallback.copyOnMiss` is set, this will copy files from the fallback
  11. // to the primary, in the event that they are missing.
  12. //
  13. // It is unlikely that the bucket/location name will be the same on the fallback
  14. // as the primary. The bucket names should be overridden in `Settings.filestore.fallback.buckets`
  15. // e.g.
  16. // Settings.filestore.fallback.buckets = {
  17. // myBucketOnS3: 'myBucketOnGCS'
  18. // }
  19. module.exports = function(primary, fallback) {
  20. function _wrapMethodOnBothPersistors(method) {
  21. return async function(bucket, key, ...moreArgs) {
  22. const fallbackBucket = _getFallbackBucket(bucket)
  23. await Promise.all([
  24. primary.promises[method](bucket, key, ...moreArgs),
  25. fallback.promises[method](fallbackBucket, key, ...moreArgs)
  26. ])
  27. }
  28. }
  29. async function getFileStreamWithFallback(bucket, key, opts) {
  30. const shouldCopy =
  31. Settings.filestore.fallback.copyOnMiss && !opts.start && !opts.end
  32. try {
  33. return await primary.promises.getFileStream(bucket, key, opts)
  34. } catch (err) {
  35. if (err instanceof NotFoundError) {
  36. const fallbackBucket = _getFallbackBucket(bucket)
  37. const fallbackStream = await fallback.promises.getFileStream(
  38. fallbackBucket,
  39. key,
  40. opts
  41. )
  42. // tee the stream to the client, and as a copy to the primary (if necessary)
  43. // start listening on both straight away so that we don't consume bytes
  44. // in one place before the other
  45. const returnStream = new Stream.PassThrough()
  46. pipeline(fallbackStream, returnStream)
  47. if (shouldCopy) {
  48. const copyStream = new Stream.PassThrough()
  49. pipeline(fallbackStream, copyStream)
  50. _copyStreamFromFallbackAndVerify(
  51. copyStream,
  52. fallbackBucket,
  53. bucket,
  54. key,
  55. key
  56. ).catch(() => {
  57. // swallow errors, as this runs in the background and will log a warning
  58. })
  59. }
  60. return returnStream
  61. }
  62. throw err
  63. }
  64. }
  65. async function copyFileWithFallback(bucket, sourceKey, destKey) {
  66. try {
  67. return await primary.promises.copyFile(bucket, sourceKey, destKey)
  68. } catch (err) {
  69. if (err instanceof NotFoundError) {
  70. const fallbackBucket = _getFallbackBucket(bucket)
  71. const fallbackStream = await fallback.promises.getFileStream(
  72. fallbackBucket,
  73. sourceKey,
  74. {}
  75. )
  76. const copyStream = new Stream.PassThrough()
  77. pipeline(fallbackStream, copyStream)
  78. if (Settings.filestore.fallback.copyOnMiss) {
  79. const missStream = new Stream.PassThrough()
  80. pipeline(fallbackStream, missStream)
  81. // copy from sourceKey -> sourceKey
  82. _copyStreamFromFallbackAndVerify(
  83. missStream,
  84. fallbackBucket,
  85. bucket,
  86. sourceKey,
  87. sourceKey
  88. ).then(() => {
  89. // swallow errors, as this runs in the background and will log a warning
  90. })
  91. }
  92. // copy from sourceKey -> destKey
  93. return _copyStreamFromFallbackAndVerify(
  94. copyStream,
  95. fallbackBucket,
  96. bucket,
  97. sourceKey,
  98. destKey
  99. )
  100. }
  101. throw err
  102. }
  103. }
  104. function _getFallbackBucket(bucket) {
  105. return Settings.filestore.fallback.buckets[bucket]
  106. }
  107. function _wrapFallbackMethod(method) {
  108. return async function(bucket, key, ...moreArgs) {
  109. try {
  110. return await primary.promises[method](bucket, key, ...moreArgs)
  111. } catch (err) {
  112. if (err instanceof NotFoundError) {
  113. const fallbackBucket = _getFallbackBucket(bucket)
  114. if (Settings.filestore.fallback.copyOnMiss) {
  115. const fallbackStream = await fallback.promises.getFileStream(
  116. fallbackBucket,
  117. key,
  118. {}
  119. )
  120. // run in background
  121. _copyStreamFromFallbackAndVerify(
  122. fallbackStream,
  123. fallbackBucket,
  124. bucket,
  125. key,
  126. key
  127. ).catch(err => {
  128. logger.warn({ err }, 'failed to copy file from fallback')
  129. })
  130. }
  131. return fallback.promises[method](fallbackBucket, key, ...moreArgs)
  132. }
  133. throw err
  134. }
  135. }
  136. }
  137. async function _copyStreamFromFallbackAndVerify(
  138. stream,
  139. sourceBucket,
  140. destBucket,
  141. sourceKey,
  142. destKey
  143. ) {
  144. try {
  145. let sourceMd5
  146. try {
  147. sourceMd5 = await fallback.promises.getFileMd5Hash(
  148. sourceBucket,
  149. sourceKey
  150. )
  151. } catch (err) {
  152. logger.warn(err, 'error getting md5 hash from fallback persistor')
  153. }
  154. await primary.promises.sendStream(destBucket, destKey, stream, sourceMd5)
  155. } catch (err) {
  156. const error = new WriteError({
  157. message: 'unable to copy file to destination persistor',
  158. info: {
  159. sourceBucket,
  160. destBucket,
  161. sourceKey,
  162. destKey
  163. }
  164. }).withCause(err)
  165. metrics.inc('fallback.copy.failure')
  166. try {
  167. await primary.promises.deleteFile(destBucket, destKey)
  168. } catch (err) {
  169. error.info.cleanupError = new WriteError({
  170. message: 'unable to clean up destination copy artifact',
  171. info: {
  172. destBucket,
  173. destKey
  174. }
  175. }).withCause(err)
  176. }
  177. logger.warn({ error }, 'failed to copy file from fallback')
  178. throw error
  179. }
  180. }
  181. return {
  182. primaryPersistor: primary,
  183. fallbackPersistor: fallback,
  184. sendFile: primary.sendFile,
  185. sendStream: primary.sendStream,
  186. getFileStream: callbackify(getFileStreamWithFallback),
  187. getFileMd5Hash: callbackify(_wrapFallbackMethod('getFileMd5Hash')),
  188. deleteDirectory: callbackify(
  189. _wrapMethodOnBothPersistors('deleteDirectory')
  190. ),
  191. getFileSize: callbackify(_wrapFallbackMethod('getFileSize')),
  192. deleteFile: callbackify(_wrapMethodOnBothPersistors('deleteFile')),
  193. copyFile: callbackify(copyFileWithFallback),
  194. checkIfFileExists: callbackify(_wrapFallbackMethod('checkIfFileExists')),
  195. directorySize: callbackify(_wrapFallbackMethod('directorySize')),
  196. promises: {
  197. sendFile: primary.promises.sendFile,
  198. sendStream: primary.promises.sendStream,
  199. getFileStream: getFileStreamWithFallback,
  200. getFileMd5Hash: _wrapFallbackMethod('getFileMd5Hash'),
  201. deleteDirectory: _wrapMethodOnBothPersistors('deleteDirectory'),
  202. getFileSize: _wrapFallbackMethod('getFileSize'),
  203. deleteFile: _wrapMethodOnBothPersistors('deleteFile'),
  204. copyFile: copyFileWithFallback,
  205. checkIfFileExists: _wrapFallbackMethod('checkIfFileExists'),
  206. directorySize: _wrapFallbackMethod('directorySize')
  207. }
  208. }
  209. }