DockerRunner.js 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695
  1. /* eslint-disable
  2. camelcase,
  3. handle-callback-err,
  4. no-return-assign,
  5. no-unused-vars,
  6. */
  7. // TODO: This file was created by bulk-decaffeinate.
  8. // Fix any style issues and re-enable lint.
  9. /*
  10. * decaffeinate suggestions:
  11. * DS101: Remove unnecessary use of Array.from
  12. * DS102: Remove unnecessary code created because of implicit returns
  13. * DS103: Rewrite code to no longer use __guard__
  14. * DS205: Consider reworking code to avoid use of IIFEs
  15. * DS207: Consider shorter variations of null checks
  16. * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  17. */
  18. let DockerRunner, oneHour
  19. const Settings = require('settings-sharelatex')
  20. const logger = require('logger-sharelatex')
  21. const Docker = require('dockerode')
  22. const dockerode = new Docker()
  23. const crypto = require('crypto')
  24. const async = require('async')
  25. const LockManager = require('./DockerLockManager')
  26. const fs = require('fs')
  27. const Path = require('path')
  28. const _ = require('underscore')
  29. logger.info('using docker runner')
  30. const usingSiblingContainers = () =>
  31. __guard__(
  32. Settings != null ? Settings.path : undefined,
  33. x => x.sandboxedCompilesHostDir
  34. ) != null
  35. let containerMonitorTimeout
  36. let containerMonitorInterval
  37. module.exports = DockerRunner = {
  38. ERR_NOT_DIRECTORY: new Error('not a directory'),
  39. ERR_TERMINATED: new Error('terminated'),
  40. ERR_EXITED: new Error('exited'),
  41. ERR_TIMED_OUT: new Error('container timed out'),
  42. run(project_id, command, directory, image, timeout, environment, callback) {
  43. let name
  44. if (callback == null) {
  45. callback = function(error, output) {}
  46. }
  47. if (usingSiblingContainers()) {
  48. const _newPath = Settings.path.sandboxedCompilesHostDir
  49. logger.log(
  50. { path: _newPath },
  51. 'altering bind path for sibling containers'
  52. )
  53. // Server Pro, example:
  54. // '/var/lib/sharelatex/data/compiles/<project-id>'
  55. // ... becomes ...
  56. // '/opt/sharelatex_data/data/compiles/<project-id>'
  57. directory = Path.join(
  58. Settings.path.sandboxedCompilesHostDir,
  59. Path.basename(directory)
  60. )
  61. }
  62. const volumes = {}
  63. volumes[directory] = '/compile'
  64. command = Array.from(command).map(arg =>
  65. __guardMethod__(arg.toString(), 'replace', o =>
  66. o.replace('$COMPILE_DIR', '/compile')
  67. )
  68. )
  69. if (image == null) {
  70. ;({ image } = Settings.clsi.docker)
  71. }
  72. if (Settings.texliveImageNameOveride != null) {
  73. const img = image.split('/')
  74. image = `${Settings.texliveImageNameOveride}/${img[2]}`
  75. }
  76. const options = DockerRunner._getContainerOptions(
  77. command,
  78. image,
  79. volumes,
  80. timeout,
  81. environment
  82. )
  83. const fingerprint = DockerRunner._fingerprintContainer(options)
  84. options.name = name = `project-${project_id}-${fingerprint}`
  85. // logOptions = _.clone(options)
  86. // logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
  87. logger.log({ project_id }, 'running docker container')
  88. DockerRunner._runAndWaitForContainer(options, volumes, timeout, function(
  89. error,
  90. output
  91. ) {
  92. if (
  93. __guard__(error != null ? error.message : undefined, x =>
  94. x.match('HTTP code is 500')
  95. )
  96. ) {
  97. logger.log(
  98. { err: error, project_id },
  99. 'error running container so destroying and retrying'
  100. )
  101. return DockerRunner.destroyContainer(name, null, true, function(error) {
  102. if (error != null) {
  103. return callback(error)
  104. }
  105. return DockerRunner._runAndWaitForContainer(
  106. options,
  107. volumes,
  108. timeout,
  109. callback
  110. )
  111. })
  112. } else {
  113. return callback(error, output)
  114. }
  115. })
  116. return name
  117. }, // pass back the container name to allow it to be killed
  118. kill(container_id, callback) {
  119. if (callback == null) {
  120. callback = function(error) {}
  121. }
  122. logger.log({ container_id }, 'sending kill signal to container')
  123. const container = dockerode.getContainer(container_id)
  124. return container.kill(function(error) {
  125. if (
  126. error != null &&
  127. __guardMethod__(error != null ? error.message : undefined, 'match', o =>
  128. o.match(/Cannot kill container .* is not running/)
  129. )
  130. ) {
  131. logger.warn(
  132. { err: error, container_id },
  133. 'container not running, continuing'
  134. )
  135. error = null
  136. }
  137. if (error != null) {
  138. logger.error({ err: error, container_id }, 'error killing container')
  139. return callback(error)
  140. } else {
  141. return callback()
  142. }
  143. })
  144. },
  145. _runAndWaitForContainer(options, volumes, timeout, _callback) {
  146. if (_callback == null) {
  147. _callback = function(error, output) {}
  148. }
  149. const callback = function(...args) {
  150. _callback(...Array.from(args || []))
  151. // Only call the callback once
  152. return (_callback = function() {})
  153. }
  154. const { name } = options
  155. let streamEnded = false
  156. let containerReturned = false
  157. let output = {}
  158. const callbackIfFinished = function() {
  159. if (streamEnded && containerReturned) {
  160. return callback(null, output)
  161. }
  162. }
  163. const attachStreamHandler = function(error, _output) {
  164. if (error != null) {
  165. return callback(error)
  166. }
  167. output = _output
  168. streamEnded = true
  169. return callbackIfFinished()
  170. }
  171. return DockerRunner.startContainer(
  172. options,
  173. volumes,
  174. attachStreamHandler,
  175. function(error, containerId) {
  176. if (error != null) {
  177. return callback(error)
  178. }
  179. return DockerRunner.waitForContainer(name, timeout, function(
  180. error,
  181. exitCode
  182. ) {
  183. let err
  184. if (error != null) {
  185. return callback(error)
  186. }
  187. if (exitCode === 137) {
  188. // exit status from kill -9
  189. err = DockerRunner.ERR_TERMINATED
  190. err.terminated = true
  191. return callback(err)
  192. }
  193. if (exitCode === 1) {
  194. // exit status from chktex
  195. err = DockerRunner.ERR_EXITED
  196. err.code = exitCode
  197. return callback(err)
  198. }
  199. containerReturned = true
  200. __guard__(
  201. options != null ? options.HostConfig : undefined,
  202. x => (x.SecurityOpt = null)
  203. ) // small log line
  204. logger.log({ err, exitCode, options }, 'docker container has exited')
  205. return callbackIfFinished()
  206. })
  207. }
  208. )
  209. },
  210. _getContainerOptions(command, image, volumes, timeout, environment) {
  211. let m, year
  212. let key, value, hostVol, dockerVol
  213. const timeoutInSeconds = timeout / 1000
  214. const dockerVolumes = {}
  215. for (hostVol in volumes) {
  216. dockerVol = volumes[hostVol]
  217. dockerVolumes[dockerVol] = {}
  218. if (volumes[hostVol].slice(-3).indexOf(':r') === -1) {
  219. volumes[hostVol] = `${dockerVol}:rw`
  220. }
  221. }
  222. // merge settings and environment parameter
  223. const env = {}
  224. for (const src of [Settings.clsi.docker.env, environment || {}]) {
  225. for (key in src) {
  226. value = src[key]
  227. env[key] = value
  228. }
  229. }
  230. // set the path based on the image year
  231. if ((m = image.match(/:([0-9]+)\.[0-9]+/))) {
  232. year = m[1]
  233. } else {
  234. year = '2014'
  235. }
  236. env.PATH = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/${year}/bin/x86_64-linux/`
  237. const options = {
  238. Cmd: command,
  239. Image: image,
  240. Volumes: dockerVolumes,
  241. WorkingDir: '/compile',
  242. NetworkDisabled: true,
  243. Memory: 1024 * 1024 * 1024 * 1024, // 1 Gb
  244. User: Settings.clsi.docker.user,
  245. Env: (() => {
  246. const result = []
  247. for (key in env) {
  248. value = env[key]
  249. result.push(`${key}=${value}`)
  250. }
  251. return result
  252. })(), // convert the environment hash to an array
  253. HostConfig: {
  254. Binds: (() => {
  255. const result1 = []
  256. for (hostVol in volumes) {
  257. dockerVol = volumes[hostVol]
  258. result1.push(`${hostVol}:${dockerVol}`)
  259. }
  260. return result1
  261. })(),
  262. LogConfig: { Type: 'none', Config: {} },
  263. Ulimits: [
  264. {
  265. Name: 'cpu',
  266. Soft: timeoutInSeconds + 5,
  267. Hard: timeoutInSeconds + 10
  268. }
  269. ],
  270. CapDrop: 'ALL',
  271. SecurityOpt: ['no-new-privileges']
  272. }
  273. }
  274. if (
  275. (Settings.path != null ? Settings.path.synctexBinHostPath : undefined) !=
  276. null
  277. ) {
  278. options.HostConfig.Binds.push(
  279. `${Settings.path.synctexBinHostPath}:/opt/synctex:ro`
  280. )
  281. }
  282. if (Settings.clsi.docker.seccomp_profile != null) {
  283. options.HostConfig.SecurityOpt.push(
  284. `seccomp=${Settings.clsi.docker.seccomp_profile}`
  285. )
  286. }
  287. return options
  288. },
  289. _fingerprintContainer(containerOptions) {
  290. // Yay, Hashing!
  291. const json = JSON.stringify(containerOptions)
  292. return crypto
  293. .createHash('md5')
  294. .update(json)
  295. .digest('hex')
  296. },
  297. startContainer(options, volumes, attachStreamHandler, callback) {
  298. return LockManager.runWithLock(
  299. options.name,
  300. releaseLock =>
  301. // Check that volumes exist before starting the container.
  302. // When a container is started with volume pointing to a
  303. // non-existent directory then docker creates the directory but
  304. // with root ownership.
  305. DockerRunner._checkVolumes(options, volumes, function(err) {
  306. if (err != null) {
  307. return releaseLock(err)
  308. }
  309. return DockerRunner._startContainer(
  310. options,
  311. volumes,
  312. attachStreamHandler,
  313. releaseLock
  314. )
  315. }),
  316. callback
  317. )
  318. },
  319. // Check that volumes exist and are directories
  320. _checkVolumes(options, volumes, callback) {
  321. if (callback == null) {
  322. callback = function(error, containerName) {}
  323. }
  324. if (usingSiblingContainers()) {
  325. // Server Pro, with sibling-containers active, skip checks
  326. return callback(null)
  327. }
  328. const checkVolume = (path, cb) =>
  329. fs.stat(path, function(err, stats) {
  330. if (err != null) {
  331. return cb(err)
  332. }
  333. if (!(stats != null ? stats.isDirectory() : undefined)) {
  334. return cb(DockerRunner.ERR_NOT_DIRECTORY)
  335. }
  336. return cb()
  337. })
  338. const jobs = []
  339. for (const vol in volumes) {
  340. ;(vol => jobs.push(cb => checkVolume(vol, cb)))(vol)
  341. }
  342. return async.series(jobs, callback)
  343. },
  344. _startContainer(options, volumes, attachStreamHandler, callback) {
  345. if (callback == null) {
  346. callback = function(error, output) {}
  347. }
  348. callback = _.once(callback)
  349. const { name } = options
  350. logger.log({ container_name: name }, 'starting container')
  351. const container = dockerode.getContainer(name)
  352. const createAndStartContainer = () =>
  353. dockerode.createContainer(options, function(error, container) {
  354. if (error != null) {
  355. return callback(error)
  356. }
  357. return startExistingContainer()
  358. })
  359. var startExistingContainer = () =>
  360. DockerRunner.attachToContainer(
  361. options.name,
  362. attachStreamHandler,
  363. function(error) {
  364. if (error != null) {
  365. return callback(error)
  366. }
  367. return container.start(function(error) {
  368. if (
  369. error != null &&
  370. (error != null ? error.statusCode : undefined) !== 304
  371. ) {
  372. // already running
  373. return callback(error)
  374. } else {
  375. return callback()
  376. }
  377. })
  378. }
  379. )
  380. return container.inspect(function(error, stats) {
  381. if ((error != null ? error.statusCode : undefined) === 404) {
  382. return createAndStartContainer()
  383. } else if (error != null) {
  384. logger.err(
  385. { container_name: name, error },
  386. 'unable to inspect container to start'
  387. )
  388. return callback(error)
  389. } else {
  390. return startExistingContainer()
  391. }
  392. })
  393. },
  394. attachToContainer(containerId, attachStreamHandler, attachStartCallback) {
  395. const container = dockerode.getContainer(containerId)
  396. return container.attach({ stdout: 1, stderr: 1, stream: 1 }, function(
  397. error,
  398. stream
  399. ) {
  400. if (error != null) {
  401. logger.error(
  402. { err: error, container_id: containerId },
  403. 'error attaching to container'
  404. )
  405. return attachStartCallback(error)
  406. } else {
  407. attachStartCallback()
  408. }
  409. logger.log({ container_id: containerId }, 'attached to container')
  410. const MAX_OUTPUT = 1024 * 1024 // limit output to 1MB
  411. const createStringOutputStream = function(name) {
  412. return {
  413. data: '',
  414. overflowed: false,
  415. write(data) {
  416. if (this.overflowed) {
  417. return
  418. }
  419. if (this.data.length < MAX_OUTPUT) {
  420. return (this.data += data)
  421. } else {
  422. logger.error(
  423. {
  424. container_id: containerId,
  425. length: this.data.length,
  426. maxLen: MAX_OUTPUT
  427. },
  428. `${name} exceeds max size`
  429. )
  430. this.data += `(...truncated at ${MAX_OUTPUT} chars...)`
  431. return (this.overflowed = true)
  432. }
  433. }
  434. // kill container if too much output
  435. // docker.containers.kill(containerId, () ->)
  436. }
  437. }
  438. const stdout = createStringOutputStream('stdout')
  439. const stderr = createStringOutputStream('stderr')
  440. container.modem.demuxStream(stream, stdout, stderr)
  441. stream.on('error', err =>
  442. logger.error(
  443. { err, container_id: containerId },
  444. 'error reading from container stream'
  445. )
  446. )
  447. return stream.on('end', () =>
  448. attachStreamHandler(null, { stdout: stdout.data, stderr: stderr.data })
  449. )
  450. })
  451. },
  452. waitForContainer(containerId, timeout, _callback) {
  453. if (_callback == null) {
  454. _callback = function(error, exitCode) {}
  455. }
  456. const callback = function(...args) {
  457. _callback(...Array.from(args || []))
  458. // Only call the callback once
  459. return (_callback = function() {})
  460. }
  461. const container = dockerode.getContainer(containerId)
  462. let timedOut = false
  463. const timeoutId = setTimeout(function() {
  464. timedOut = true
  465. logger.log(
  466. { container_id: containerId },
  467. 'timeout reached, killing container'
  468. )
  469. return container.kill(function() {})
  470. }, timeout)
  471. logger.log({ container_id: containerId }, 'waiting for docker container')
  472. return container.wait(function(error, res) {
  473. if (error != null) {
  474. clearTimeout(timeoutId)
  475. logger.error(
  476. { err: error, container_id: containerId },
  477. 'error waiting for container'
  478. )
  479. return callback(error)
  480. }
  481. if (timedOut) {
  482. logger.log({ containerId }, 'docker container timed out')
  483. error = DockerRunner.ERR_TIMED_OUT
  484. error.timedout = true
  485. return callback(error)
  486. } else {
  487. clearTimeout(timeoutId)
  488. logger.log(
  489. { container_id: containerId, exitCode: res.StatusCode },
  490. 'docker container returned'
  491. )
  492. return callback(null, res.StatusCode)
  493. }
  494. })
  495. },
  496. destroyContainer(containerName, containerId, shouldForce, callback) {
  497. // We want the containerName for the lock and, ideally, the
  498. // containerId to delete. There is a bug in the docker.io module
  499. // where if you delete by name and there is an error, it throws an
  500. // async exception, but if you delete by id it just does a normal
  501. // error callback. We fall back to deleting by name if no id is
  502. // supplied.
  503. if (callback == null) {
  504. callback = function(error) {}
  505. }
  506. return LockManager.runWithLock(
  507. containerName,
  508. releaseLock =>
  509. DockerRunner._destroyContainer(
  510. containerId || containerName,
  511. shouldForce,
  512. releaseLock
  513. ),
  514. callback
  515. )
  516. },
  517. _destroyContainer(containerId, shouldForce, callback) {
  518. if (callback == null) {
  519. callback = function(error) {}
  520. }
  521. logger.log({ container_id: containerId }, 'destroying docker container')
  522. const container = dockerode.getContainer(containerId)
  523. return container.remove({ force: shouldForce === true }, function(error) {
  524. if (
  525. error != null &&
  526. (error != null ? error.statusCode : undefined) === 404
  527. ) {
  528. logger.warn(
  529. { err: error, container_id: containerId },
  530. 'container not found, continuing'
  531. )
  532. error = null
  533. }
  534. if (error != null) {
  535. logger.error(
  536. { err: error, container_id: containerId },
  537. 'error destroying container'
  538. )
  539. } else {
  540. logger.log({ container_id: containerId }, 'destroyed container')
  541. }
  542. return callback(error)
  543. })
  544. },
  545. // handle expiry of docker containers
  546. MAX_CONTAINER_AGE:
  547. Settings.clsi.docker.maxContainerAge || (oneHour = 60 * 60 * 1000),
  548. examineOldContainer(container, callback) {
  549. if (callback == null) {
  550. callback = function(error, name, id, ttl) {}
  551. }
  552. const name =
  553. container.Name ||
  554. (container.Names != null ? container.Names[0] : undefined)
  555. const created = container.Created * 1000 // creation time is returned in seconds
  556. const now = Date.now()
  557. const age = now - created
  558. const maxAge = DockerRunner.MAX_CONTAINER_AGE
  559. const ttl = maxAge - age
  560. logger.log(
  561. { containerName: name, created, now, age, maxAge, ttl },
  562. 'checking whether to destroy container'
  563. )
  564. return callback(null, name, container.Id, ttl)
  565. },
  566. destroyOldContainers(callback) {
  567. if (callback == null) {
  568. callback = function(error) {}
  569. }
  570. return dockerode.listContainers({ all: true }, function(error, containers) {
  571. if (error != null) {
  572. return callback(error)
  573. }
  574. const jobs = []
  575. for (const container of Array.from(containers || [])) {
  576. ;(container =>
  577. DockerRunner.examineOldContainer(container, function(
  578. err,
  579. name,
  580. id,
  581. ttl
  582. ) {
  583. if (name.slice(0, 9) === '/project-' && ttl <= 0) {
  584. return jobs.push(cb =>
  585. DockerRunner.destroyContainer(name, id, false, () => cb())
  586. )
  587. }
  588. }))(container)
  589. }
  590. // Ignore errors because some containers get stuck but
  591. // will be destroyed next time
  592. return async.series(jobs, callback)
  593. })
  594. },
  595. startContainerMonitor() {
  596. logger.log(
  597. { maxAge: DockerRunner.MAX_CONTAINER_AGE },
  598. 'starting container expiry'
  599. )
  600. // guarantee only one monitor is running
  601. DockerRunner.stopContainerMonitor()
  602. // randomise the start time
  603. const randomDelay = Math.floor(Math.random() * 5 * 60 * 1000)
  604. containerMonitorTimeout = setTimeout(() => {
  605. containerMonitorInterval = setInterval(
  606. () => DockerRunner.destroyOldContainers(),
  607. (oneHour = 60 * 60 * 1000)
  608. )
  609. }, randomDelay)
  610. },
  611. stopContainerMonitor() {
  612. if (containerMonitorTimeout) {
  613. clearTimeout(containerMonitorTimeout)
  614. containerMonitorTimeout = undefined
  615. }
  616. if (containerMonitorInterval) {
  617. clearInterval(containerMonitorTimeout)
  618. containerMonitorTimeout = undefined
  619. }
  620. }
  621. }
  622. DockerRunner.startContainerMonitor()
  623. function __guard__(value, transform) {
  624. return typeof value !== 'undefined' && value !== null
  625. ? transform(value)
  626. : undefined
  627. }
  628. function __guardMethod__(obj, methodName, transform) {
  629. if (
  630. typeof obj !== 'undefined' &&
  631. obj !== null &&
  632. typeof obj[methodName] === 'function'
  633. ) {
  634. return transform(obj, methodName)
  635. } else {
  636. return undefined
  637. }
  638. }