ProjectManager.js 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311
  1. const RedisManager = require('./RedisManager')
  2. const ProjectHistoryRedisManager = require('./ProjectHistoryRedisManager')
  3. const DocumentManager = require('./DocumentManager')
  4. const HistoryManager = require('./HistoryManager')
  5. const async = require('async')
  6. const logger = require('logger-sharelatex')
  7. const Metrics = require('./Metrics')
  8. const Errors = require('./Errors')
  9. module.exports = {
  10. flushProjectWithLocks(projectId, _callback) {
  11. const timer = new Metrics.Timer('projectManager.flushProjectWithLocks')
  12. const callback = function (...args) {
  13. timer.done()
  14. _callback(...args)
  15. }
  16. RedisManager.getDocIdsInProject(projectId, (error, docIds) => {
  17. if (error) {
  18. return callback(error)
  19. }
  20. const errors = []
  21. const jobs = docIds.map((docId) => (callback) => {
  22. DocumentManager.flushDocIfLoadedWithLock(projectId, docId, (error) => {
  23. if (error instanceof Errors.NotFoundError) {
  24. logger.warn(
  25. { err: error, projectId, docId },
  26. 'found deleted doc when flushing'
  27. )
  28. callback()
  29. } else if (error) {
  30. logger.error({ err: error, projectId, docId }, 'error flushing doc')
  31. errors.push(error)
  32. callback()
  33. } else {
  34. callback()
  35. }
  36. })
  37. })
  38. logger.log({ projectId, docIds }, 'flushing docs')
  39. async.series(jobs, () => {
  40. if (errors.length > 0) {
  41. callback(new Error('Errors flushing docs. See log for details'))
  42. } else {
  43. callback(null)
  44. }
  45. })
  46. })
  47. },
  48. flushAndDeleteProjectWithLocks(projectId, options, _callback) {
  49. const timer = new Metrics.Timer(
  50. 'projectManager.flushAndDeleteProjectWithLocks'
  51. )
  52. const callback = function (...args) {
  53. timer.done()
  54. _callback(...args)
  55. }
  56. RedisManager.getDocIdsInProject(projectId, (error, docIds) => {
  57. if (error) {
  58. return callback(error)
  59. }
  60. const errors = []
  61. const jobs = docIds.map((docId) => (callback) => {
  62. DocumentManager.flushAndDeleteDocWithLock(
  63. projectId,
  64. docId,
  65. {},
  66. (error) => {
  67. if (error) {
  68. logger.error(
  69. { err: error, projectId, docId },
  70. 'error deleting doc'
  71. )
  72. errors.push(error)
  73. }
  74. callback()
  75. }
  76. )
  77. })
  78. logger.log({ projectId, docIds }, 'deleting docs')
  79. async.series(jobs, () =>
  80. // When deleting the project here we want to ensure that project
  81. // history is completely flushed because the project may be
  82. // deleted in web after this call completes, and so further
  83. // attempts to flush would fail after that.
  84. HistoryManager.flushProjectChanges(projectId, options, (error) => {
  85. if (errors.length > 0) {
  86. callback(new Error('Errors deleting docs. See log for details'))
  87. } else if (error) {
  88. callback(error)
  89. } else {
  90. callback(null)
  91. }
  92. })
  93. )
  94. })
  95. },
  96. queueFlushAndDeleteProject(projectId, callback) {
  97. RedisManager.queueFlushAndDeleteProject(projectId, (error) => {
  98. if (error) {
  99. logger.error(
  100. { projectId, error },
  101. 'error adding project to flush and delete queue'
  102. )
  103. return callback(error)
  104. }
  105. Metrics.inc('queued-delete')
  106. callback()
  107. })
  108. },
  109. getProjectDocsTimestamps(projectId, callback) {
  110. RedisManager.getDocIdsInProject(projectId, (error, docIds) => {
  111. if (error) {
  112. return callback(error)
  113. }
  114. if (docIds.length === 0) {
  115. return callback(null, [])
  116. }
  117. RedisManager.getDocTimestamps(docIds, (error, timestamps) => {
  118. if (error) {
  119. return callback(error)
  120. }
  121. callback(null, timestamps)
  122. })
  123. })
  124. },
  125. getProjectDocsAndFlushIfOld(
  126. projectId,
  127. projectStateHash,
  128. excludeVersions,
  129. _callback
  130. ) {
  131. const timer = new Metrics.Timer(
  132. 'projectManager.getProjectDocsAndFlushIfOld'
  133. )
  134. const callback = function (...args) {
  135. timer.done()
  136. _callback(...args)
  137. }
  138. RedisManager.checkOrSetProjectState(
  139. projectId,
  140. projectStateHash,
  141. (error, projectStateChanged) => {
  142. if (error) {
  143. logger.error(
  144. { err: error, projectId },
  145. 'error getting/setting project state in getProjectDocsAndFlushIfOld'
  146. )
  147. return callback(error)
  148. }
  149. // we can't return docs if project structure has changed
  150. if (projectStateChanged) {
  151. return callback(
  152. Errors.ProjectStateChangedError('project state changed')
  153. )
  154. }
  155. // project structure hasn't changed, return doc content from redis
  156. RedisManager.getDocIdsInProject(projectId, (error, docIds) => {
  157. if (error) {
  158. logger.error(
  159. { err: error, projectId },
  160. 'error getting doc ids in getProjectDocs'
  161. )
  162. return callback(error)
  163. }
  164. // get the doc lines from redis
  165. const jobs = docIds.map((docId) => (cb) => {
  166. DocumentManager.getDocAndFlushIfOldWithLock(
  167. projectId,
  168. docId,
  169. (err, lines, version) => {
  170. if (err) {
  171. logger.error(
  172. { err, projectId, docId },
  173. 'error getting project doc lines in getProjectDocsAndFlushIfOld'
  174. )
  175. return cb(err)
  176. }
  177. const doc = { _id: docId, lines, v: version } // create a doc object to return
  178. cb(null, doc)
  179. }
  180. )
  181. })
  182. async.series(jobs, (error, docs) => {
  183. if (error) {
  184. return callback(error)
  185. }
  186. callback(null, docs)
  187. })
  188. })
  189. }
  190. )
  191. },
  192. clearProjectState(projectId, callback) {
  193. RedisManager.clearProjectState(projectId, callback)
  194. },
  195. updateProjectWithLocks(
  196. projectId,
  197. projectHistoryId,
  198. userId,
  199. docUpdates,
  200. fileUpdates,
  201. version,
  202. _callback
  203. ) {
  204. const timer = new Metrics.Timer('projectManager.updateProject')
  205. const callback = function (...args) {
  206. timer.done()
  207. _callback(...args)
  208. }
  209. const projectVersion = version
  210. let projectSubversion = 0 // project versions can have multiple operations
  211. let projectOpsLength = 0
  212. const handleDocUpdate = function (projectUpdate, cb) {
  213. const docId = projectUpdate.id
  214. projectUpdate.version = `${projectVersion}.${projectSubversion++}`
  215. if (projectUpdate.docLines != null) {
  216. ProjectHistoryRedisManager.queueAddEntity(
  217. projectId,
  218. projectHistoryId,
  219. 'doc',
  220. docId,
  221. userId,
  222. projectUpdate,
  223. (error, count) => {
  224. projectOpsLength = count
  225. cb(error)
  226. }
  227. )
  228. } else {
  229. DocumentManager.renameDocWithLock(
  230. projectId,
  231. docId,
  232. userId,
  233. projectUpdate,
  234. projectHistoryId,
  235. (error, count) => {
  236. projectOpsLength = count
  237. cb(error)
  238. }
  239. )
  240. }
  241. }
  242. const handleFileUpdate = function (projectUpdate, cb) {
  243. const fileId = projectUpdate.id
  244. projectUpdate.version = `${projectVersion}.${projectSubversion++}`
  245. if (projectUpdate.url != null) {
  246. ProjectHistoryRedisManager.queueAddEntity(
  247. projectId,
  248. projectHistoryId,
  249. 'file',
  250. fileId,
  251. userId,
  252. projectUpdate,
  253. (error, count) => {
  254. projectOpsLength = count
  255. cb(error)
  256. }
  257. )
  258. } else {
  259. ProjectHistoryRedisManager.queueRenameEntity(
  260. projectId,
  261. projectHistoryId,
  262. 'file',
  263. fileId,
  264. userId,
  265. projectUpdate,
  266. (error, count) => {
  267. projectOpsLength = count
  268. cb(error)
  269. }
  270. )
  271. }
  272. }
  273. async.eachSeries(docUpdates, handleDocUpdate, (error) => {
  274. if (error) {
  275. return callback(error)
  276. }
  277. async.eachSeries(fileUpdates, handleFileUpdate, (error) => {
  278. if (error) {
  279. return callback(error)
  280. }
  281. if (
  282. HistoryManager.shouldFlushHistoryOps(
  283. projectOpsLength,
  284. docUpdates.length + fileUpdates.length,
  285. HistoryManager.FLUSH_PROJECT_EVERY_N_OPS
  286. )
  287. ) {
  288. HistoryManager.flushProjectChangesAsync(projectId)
  289. }
  290. callback()
  291. })
  292. })
  293. },
  294. }