Refactor the LinkedFiles/Agent system, and track build_id for output files

services/web/app/coffee/Features/Compile/ClsiManager.coffee

@@ -182,8 +182,8 @@ module.exports = ClsiManager =
 				return callback(error) if error?
 				callback(null, projectStateHash, docs)
 
-	getOutputFileStream: (project_id, output_file_path, callback=(err, readStream)->) ->
-		url = "#{Settings.apis.clsi.url}/project/#{project_id}/output/#{output_file_path}"
+	getOutputFileStream: (project_id, user_id, build_id, output_file_path, callback=(err, readStream)->) ->
+		url = "#{Settings.apis.clsi.url}/project/#{project_id}/user/#{user_id}/build/#{build_id}/output/#{output_file_path}"
 		ClsiCookieManager.getCookieJar project_id, (err, jar)->
 			return callback(err) if err?
 			options = { url: url, method: "GET", timeout: 60 * 1000, jar : jar }

services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee

@@ -4,8 +4,11 @@ ProjectLocator = require '../Project/ProjectLocator'
 Settings = require 'settings-sharelatex'
 logger = require 'logger-sharelatex'
 _ = require 'underscore'
+LinkedFilesErrors = require './LinkedFilesErrors'
+
 
 module.exports = LinkedFilesController = {
+
 	Agents: {
 		url: require('./UrlAgent'),
 		project_file: require('./ProjectFileAgent'),
@@ -38,16 +41,16 @@ module.exports = LinkedFilesController = {
 		if !Agent?
 			return res.sendStatus(400)
 
-		linkedFileData = Agent.sanitizeData(data)
-		linkedFileData.provider = provider
-
-		if !Agent.canCreate(linkedFileData)
-			return res.status(403).send('Cannot create linked file')
+		data.provider = provider
 
-		LinkedFilesController._doImport(
-			req, res, next, Agent, project_id, user_id,
-			parent_folder_id, name, linkedFileData
-		)
+		Agent.createLinkedFile project_id,
+			data,
+			name,
+			parent_folder_id,
+			user_id,
+			(err, newFileId) ->
+				return LinkedFilesErrors.handleError(err, req, res, next) if err?
+				res.json(new_file_id: newFileId)
 
 	refreshLinkedFile: (req, res, next) ->
 		{project_id, file_id} = req.params
@@ -66,37 +69,14 @@ module.exports = LinkedFilesController = {
 			Agent = LinkedFilesController._getAgent(provider)
 			if !Agent?
 				return res.sendStatus(400)
-			LinkedFilesController._doImport(
-				req, res, next, Agent, project_id, user_id,
-				parent_folder_id, name, linkedFileData
-			)
 
-	_doImport: (req, res, next, Agent, project_id, user_id, parent_folder_id, name, linkedFileData) ->
-		Agent.checkAuth project_id, linkedFileData, user_id, (err, allowed) ->
-			return Agent.handleError(err, req, res, next) if err?
-			return res.sendStatus(403) if !allowed
-			Agent.decorateLinkedFileData linkedFileData, (err, newLinkedFileData) ->
-				return Agent.handleError(err) if err?
-				linkedFileData = newLinkedFileData
-				Agent.writeIncomingFileToDisk project_id,
-					linkedFileData,
-					user_id,
-					(error, fsPath) ->
-						if error?
-							logger.error(
-								{err: error, project_id, name, linkedFileData, parent_folder_id, user_id},
-								'error writing linked file to disk'
-							)
-							return Agent.handleError(error, req, res, next)
-						EditorController.upsertFile project_id,
-							parent_folder_id,
-							name,
-							fsPath,
-							linkedFileData,
-							"upload",
-							user_id,
-							(error, file) ->
-								return next(error) if error?
-								res.json(new_file_id: file._id) # created
+			Agent.refreshLinkedFile project_id,
+				linkedFileData,
+				name,
+				parent_folder_id,
+				user_id,
+				(err, newFileId) ->
+					return LinkedFilesErrors.handleError(err, req, res, next) if err?
+					res.json(new_file_id: newFileId)
 
 	}

services/web/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee

@@ -0,0 +1,122 @@
+UrlFetchFailedError = (message) ->
+	error = new Error(message)
+	error.name = 'UrlFetchFailedError'
+	error.__proto__ = UrlFetchFailedError.prototype
+	return error
+UrlFetchFailedError.prototype.__proto__ = Error.prototype
+
+
+InvalidUrlError = (message) ->
+	error = new Error(message)
+	error.name = 'InvalidUrlError'
+	error.__proto__ = InvalidUrlError.prototype
+	return error
+InvalidUrlError.prototype.__proto__ = Error.prototype
+
+
+OutputFileFetchFailedError = (message) ->
+	error = new Error(message)
+	error.name = 'OutputFileFetchFailedError'
+	error.__proto__ = OutputFileFetchFailedError.prototype
+	return error
+OutputFileFetchFailedError.prototype.__proto__ = Error.prototype
+
+
+AccessDeniedError = (message) ->
+	error = new Error(message)
+	error.name = 'AccessDenied'
+	error.__proto__ = AccessDeniedError.prototype
+	return error
+AccessDeniedError.prototype.__proto__ = Error.prototype
+
+
+BadEntityTypeError = (message) ->
+	error = new Error(message)
+	error.name = 'BadEntityType'
+	error.__proto__ = BadEntityTypeError.prototype
+	return error
+BadEntityTypeError.prototype.__proto__ = Error.prototype
+
+
+BadDataError = (message) ->
+	error = new Error(message)
+	error.name = 'BadData'
+	error.__proto__ = BadDataError.prototype
+	return error
+BadDataError.prototype.__proto__ = Error.prototype
+
+
+ProjectNotFoundError = (message) ->
+	error = new Error(message)
+	error.name = 'ProjectNotFound'
+	error.__proto__ = ProjectNotFoundError.prototype
+	return error
+ProjectNotFoundError.prototype.__proto__ = Error.prototype
+
+
+V1ProjectNotFoundError = (message) ->
+	error = new Error(message)
+	error.name = 'V1ProjectNotFound'
+	error.__proto__ = V1ProjectNotFoundError.prototype
+	return error
+V1ProjectNotFoundError.prototype.__proto__ = Error.prototype
+
+
+SourceFileNotFoundError = (message) ->
+	error = new Error(message)
+	error.name = 'SourceFileNotFound'
+	error.__proto__ = SourceFileNotFoundError.prototype
+	return error
+SourceFileNotFoundError.prototype.__proto__ = Error.prototype
+
+
+module.exports = {
+
+	UrlFetchFailedError,
+	InvalidUrlError,
+	OutputFileFetchFailedError,
+	AccessDeniedError,
+	BadEntityTypeError,
+	BadDataError,
+	ProjectNotFoundError,
+	V1ProjectNotFoundError,
+	SourceFileNotFoundError,
+
+	handleError: (error, req, res, next) ->
+		if error instanceof BadDataError
+			res.status(400).send("The submitted data is not valid")
+
+		else if error instanceof AccessDeniedError
+			res.status(403).send("You do not have access to this project")
+
+		else if error instanceof BadDataError
+			res.status(400).send("The submitted data is not valid")
+
+		else if error instanceof BadEntityTypeError
+			res.status(400).send("The file is the wrong type")
+
+		else if error instanceof SourceFileNotFoundError
+			res.status(404).send("Source file not found")
+
+		else if error instanceof ProjectNotFoundError
+			res.status(404).send("Project not found")
+
+		else if error instanceof V1ProjectNotFoundError
+			res.status(409).send("Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file")
+
+		else if error instanceof OutputFileFetchFailedError
+			res.status(404).send("Could not get output file")
+
+		else if error instanceof UrlFetchFailedError
+			res.status(422).send(
+				"Your URL could not be reached (#{error.statusCode} status code). Please check it and try again."
+			)
+
+		else if error instanceof InvalidUrlError
+			res.status(422).send(
+				"Your URL is not valid. Please check it and try again."
+			)
+
+		else
+			next(error)
+}

services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee

@@ -0,0 +1,53 @@
+LinkedFilesErrors = require './LinkedFilesErrors'
+FileWriter = require '../../infrastructure/FileWriter'
+EditorController = require '../Editor/EditorController'
+_ = require 'underscore'
+
+
+module.exports = LinkedFilesHandler =
+
+	importFromStream: (
+		project_id,
+		readStream,
+		linkedFileData,
+		name,
+		parent_folder_id,
+		user_id,
+		callback=(err, file)->
+	) ->
+		callback = _.once(callback)
+		FileWriter.writeStreamToDisk project_id, readStream, (err, fsPath) ->
+			return callback(err) if err?
+			EditorController.upsertFile project_id,
+				parent_folder_id,
+				name,
+				fsPath,
+				linkedFileData,
+				"upload",
+				user_id,
+				(err, file) =>
+					return callback(err) if err?
+					callback(null, file)
+
+	importContent: (
+		project_id,
+		content,
+		linkedFileData,
+		name,
+		parent_folder_id,
+		user_id,
+		callback=(err, file)->
+	) ->
+		callback = _.once(callback)
+		FileWriter.writeContentToDisk project_id, content, (err, fsPath) ->
+			return callback(err) if err?
+			EditorController.upsertFile project_id,
+				parent_folder_id,
+				name,
+				fsPath,
+				linkedFileData,
+				"upload",
+				user_id,
+				(err, file) =>
+					return callback(err) if err?
+					callback(null, file)

services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee

@@ -1,75 +1,97 @@
-FileWriter = require('../../infrastructure/FileWriter')
 AuthorizationManager = require('../Authorization/AuthorizationManager')
 ProjectLocator = require('../Project/ProjectLocator')
 ProjectGetter = require('../Project/ProjectGetter')
 Project = require("../../models/Project").Project
 DocstoreManager = require('../Docstore/DocstoreManager')
 FileStoreHandler = require('../FileStore/FileStoreHandler')
-FileWriter = require('../../infrastructure/FileWriter')
 _ = require "underscore"
 Settings = require 'settings-sharelatex'
+LinkedFilesHandler = require './LinkedFilesHandler'
+{
+	BadDataError,
+	AccessDeniedError,
+	BadEntityTypeError,
+	SourceFileNotFoundError,
+	ProjectNotFoundError,
+	V1ProjectNotFoundError
+} = require './LinkedFilesErrors'
 
+module.exports = ProjectFileAgent = {
 
-AccessDeniedError = (message) ->
-	error = new Error(message)
-	error.name = 'AccessDenied'
-	error.__proto__ = AccessDeniedError.prototype
-	return error
-AccessDeniedError.prototype.__proto__ = Error.prototype
-
-
-BadEntityTypeError = (message) ->
-	error = new Error(message)
-	error.name = 'BadEntityType'
-	error.__proto__ = BadEntityTypeError.prototype
-	return error
-BadEntityTypeError.prototype.__proto__ = Error.prototype
-
-
-BadDataError = (message) ->
-	error = new Error(message)
-	error.name = 'BadData'
-	error.__proto__ = BadDataError.prototype
-	return error
-BadDataError.prototype.__proto__ = Error.prototype
-
-
-ProjectNotFoundError = (message) ->
-	error = new Error(message)
-	error.name = 'ProjectNotFound'
-	error.__proto__ = ProjectNotFoundError.prototype
-	return error
-ProjectNotFoundError.prototype.__proto__ = Error.prototype
-
-
-V1ProjectNotFoundError = (message) ->
-	error = new Error(message)
-	error.name = 'V1ProjectNotFound'
-	error.__proto__ = V1ProjectNotFoundError.prototype
-	return error
-V1ProjectNotFoundError.prototype.__proto__ = Error.prototype
-
-
-SourceFileNotFoundError = (message) ->
-	error = new Error(message)
-	error.name = 'SourceFileNotFound'
-	error.__proto__ = SourceFileNotFoundError.prototype
-	return error
-SourceFileNotFoundError.prototype.__proto__ = Error.prototype
-
+	createLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+		if !@_canCreate(linkedFileData)
+			return callback(new AccessDeniedError())
+		@_go(project_id, linkedFileData, name, parent_folder_id, user_id, callback)
 
-module.exports = ProjectFileAgent = {
+	refreshLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+		@_go project_id, linkedFileData, name, parent_folder_id, user_id, callback
 
-	V1ProjectNotFoundError
-	BadDataError
-	ProjectNotFoundError
-	V1ProjectNotFoundError
+	_prepare: (project_id, linkedFileData, user_id, callback=(err, linkedFileData)->) ->
+		@_checkAuth project_id, linkedFileData, user_id, (err, allowed) =>
+			return callback(err) if err?
+			return callback(new AccessDeniedError()) if !allowed
+			@_decorateLinkedFileData linkedFileData, (err, newLinkedFileData) =>
+				return callback(err) if err?
+				if !@_validate(newLinkedFileData)
+					return callback(new BadDataError())
+				callback(null, newLinkedFileData)
 
-	_v1ProjectNotFoundMessage: "Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file"
+	_go: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+		linkedFileData = @_sanitizeData(linkedFileData)
+		@_prepare project_id, linkedFileData, user_id, (err, linkedFileData) =>
+			return callback(err) if err?
+			if !@_validate(linkedFileData)
+				return callback(new BadDataError())
+			@_getEntity linkedFileData, user_id, (err, source_project, entity, type) =>
+				return callback(err) if err?
+				if type == 'doc'
+					DocstoreManager.getDoc source_project._id, entity._id, (err, lines) ->
+						return callback(err) if err?
+						LinkedFilesHandler.importContent project_id,
+							lines.join('\n'),
+							linkedFileData,
+							name,
+							parent_folder_id,
+							user_id,
+							(err, file) ->
+								return callback(err) if err?
+								callback(null, file._id) # Created
+				else if type == 'file'
+					FileStoreHandler.getFileStream source_project._id, entity._id, null, (err, fileStream) ->
+						return callback(err) if err?
+						LinkedFilesHandler.importFromStream project_id,
+							fileStream,
+							linkedFileData,
+							name,
+							parent_folder_id,
+							user_id,
+							(err, file) ->
+								return callback(err) if err?
+								callback(null, file._id) # Created
+				else
+					callback(new BadEntityTypeError())
+
+	_getEntity:
+		(linkedFileData, current_user_id, callback = (err, entity, type) ->) ->
+			callback = _.once(callback)
+			{ source_entity_path } = linkedFileData
+			@_getSourceProject linkedFileData, (err, project) ->
+				return callback(err) if err?
+				source_project_id = project._id
+				ProjectLocator.findElementByPath {
+					project_id: source_project_id,
+					path: source_entity_path
+				}, (err, entity, type) ->
+					if err?
+						if err.toString().match(/^not found.*/)
+							err = new SourceFileNotFoundError()
+						return callback(err)
+					callback(null, project, entity, type)
 
-	sanitizeData: (data) ->
+	_sanitizeData: (data) ->
 		return _.pick(
 			data,
+			'provider',
 			'source_project_id',
 			'v1_source_doc_id',
 			'source_entity_path'
@@ -81,7 +103,7 @@ module.exports = ProjectFileAgent = {
 			data.source_entity_path?
 		)
 
-	canCreate: (data) ->
+	_canCreate: (data) ->
 		# Don't allow creation of linked-files with v1 doc ids
 		!data.v1_source_doc_id?
 
@@ -102,13 +124,13 @@ module.exports = ProjectFileAgent = {
 		else
 			callback(new BadDataError('neither v1 nor v2 id present'))
 
-	decorateLinkedFileData: (data, callback = (err, newData) ->) ->
+	_decorateLinkedFileData: (data, callback = (err, newData) ->) ->
 		callback = _.once(callback)
 		@_getSourceProject data, (err, project) ->
 			return callback(err) if err?
 			callback(err, _.extend(data, {source_project_display_name: project.name}))
 
-	checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) ->
+	_checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) ->
 		callback = _.once(callback)
 		if !ProjectFileAgent._validate(data)
 			return callback(new BadDataError())
@@ -117,53 +139,4 @@ module.exports = ProjectFileAgent = {
 			AuthorizationManager.canUserReadProject current_user_id, project._id, null, (err, canRead) ->
 				return callback(err) if err?
 				callback(null, canRead)
-
-	writeIncomingFileToDisk:
-		(project_id, data, current_user_id, callback = (error, fsPath) ->) ->
-			callback = _.once(callback)
-			if !ProjectFileAgent._validate(data)
-				return callback(new BadDataError())
-			{ source_entity_path } = data
-			@_getSourceProject data, (err, project) ->
-				return callback(err) if err?
-				source_project_id = project._id
-				ProjectLocator.findElementByPath {
-					project_id: source_project_id,
-					path: source_entity_path
-				}, (err, entity, type) ->
-					if err?
-						if err.toString().match(/^not found.*/)
-							err = new SourceFileNotFoundError()
-						return callback(err)
-					ProjectFileAgent._writeEntityToDisk source_project_id, entity._id, type, callback
-
-	_writeEntityToDisk: (project_id, entity_id, type, callback=(err, location)->) ->
-		callback = _.once(callback)
-		if type == 'doc'
-			DocstoreManager.getDoc project_id, entity_id, (err, lines) ->
-				return callback(err) if err?
-				FileWriter.writeLinesToDisk entity_id, lines, callback
-		else if type == 'file'
-			FileStoreHandler.getFileStream project_id, entity_id, null, (err, fileStream) ->
-				return callback(err) if err?
-				FileWriter.writeStreamToDisk entity_id, fileStream, callback
-		else
-			callback(new BadEntityTypeError())
-
-	handleError: (error, req, res, next) ->
-		if error instanceof AccessDeniedError
-			res.status(403).send("You do not have access to this project")
-		else if error instanceof BadDataError
-			res.status(400).send("The submitted data is not valid")
-		else if error instanceof BadEntityTypeError
-			res.status(400).send("The file is the wrong type")
-		else if error instanceof SourceFileNotFoundError
-			res.status(404).send("Source file not found")
-		else if error instanceof ProjectNotFoundError
-			res.status(404).send("Project not found")
-		else if error instanceof V1ProjectNotFoundError
-			res.status(409).send(ProjectFileAgent._v1ProjectNotFoundMessage)
-		else
-			next(error)
-		next()
 }

services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee

@@ -1,86 +1,157 @@
-FileWriter = require('../../infrastructure/FileWriter')
 AuthorizationManager = require('../Authorization/AuthorizationManager')
 ProjectGetter = require('../Project/ProjectGetter')
-FileWriter = require('../../infrastructure/FileWriter')
 Settings = require 'settings-sharelatex'
 CompileManager = require '../Compile/CompileManager'
-ClsiCookieManager = require '../Compile/ClsiCookieManager'
 ClsiManager = require '../Compile/ClsiManager'
 ProjectFileAgent = require './ProjectFileAgent'
 _ = require "underscore"
-request = require "request"
+LinkedFilesErrors = require './LinkedFilesErrors'
+LinkedFilesHandler = require './LinkedFilesHandler'
+logger = require 'logger-sharelatex'
 
 
-OutputFileFetchFailedError = (message) ->
-	error = new Error(message)
-	error.name = 'OutputFileFetchFailedError'
-	error.__proto__ = OutputFileFetchFailedError.prototype
-	return error
-OutputFileFetchFailedError.prototype.__proto__ = Error.prototype
+module.exports = ProjectOutputFileAgent = {
 
+	_prepare: (project_id, linkedFileData, user_id, callback=(err, linkedFileData)->) ->
+		@_checkAuth project_id, linkedFileData, user_id, (err, allowed) =>
+			return callback(err) if err?
+			return callback(new LinkedFilesErrors.AccessDeniedError()) if !allowed
+			@_decorateLinkedFileData linkedFileData, (err, newLinkedFileData) =>
+				return callback(err) if err?
+				if !@_validate(newLinkedFileData)
+					return callback(new BadDataError())
+				callback(null, newLinkedFileData)
 
-module.exports = ProjectOutputFileAgent = {
+	createLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+		if !@_canCreate(linkedFileData)
+			return callback(new LinkedFilesErrors.AccessDeniedError())
+		linkedFileData = @_sanitizeData(linkedFileData)
+		@_prepare project_id, linkedFileData, user_id, (err, linkedFileData) =>
+			return callback(err) if err?
+			@_getFileStream linkedFileData, user_id, (err, readStream) =>
+				return callback(err) if err?
+				readStream.on "error", callback
+				readStream.on "response", (response) =>
+					if 200 <= response.statusCode < 300
+						readStream.resume()
+						LinkedFilesHandler.importFromStream project_id,
+							readStream,
+							linkedFileData,
+							name,
+							parent_folder_id,
+							user_id,
+							(err, file) ->
+								return callback(err) if err?
+								callback(null, file._id) # Created
+					else
+						err = new LinkedFilesErrors.OutputFileFetchFailedError(
+							"Output file fetch failed: #{linkedFileData.build_id}, #{linkedFileData.source_output_file_path}"
+						)
+						err.statusCode = response.statusCode
+						callback(err)
+
+	refreshLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+		@_prepare project_id, linkedFileData, user_id, (err, linkedFileData) =>
+			return callback(err) if err?
+			@_compileAndGetFileStream linkedFileData, user_id, (err, readStream, new_build_id) =>
+				return callback(err) if err?
+				readStream.on "error", callback
+				readStream.on "response", (response) =>
+					if 200 <= response.statusCode < 300
+						readStream.resume()
+						linkedFileData.build_id = new_build_id
+						LinkedFilesHandler.importFromStream project_id,
+							readStream,
+							linkedFileData,
+							name,
+							parent_folder_id,
+							user_id,
+							(err, file) ->
+								return callback(err) if err?
+								callback(null, file._id) # Created
+					else
+						err = new LinkedFilesErrors.OutputFileFetchFailedError(
+							"Output file fetch failed: #{linkedFileData.build_id}, #{linkedFileData.source_output_file_path}"
+						)
+						err.statusCode = response.statusCode
+						callback(err)
 
-	sanitizeData: (data) ->
+
+	_sanitizeData: (data) ->
 		return {
+			provider: data.provider,
 			source_project_id: data.source_project_id,
-			source_output_file_path: data.source_output_file_path
+			source_output_file_path: data.source_output_file_path,
+			build_id: data.build_id
 		}
 
-	canCreate: ProjectFileAgent.canCreate
+	_canCreate: ProjectFileAgent._canCreate
 
 	_getSourceProject: ProjectFileAgent._getSourceProject
 
-	decorateLinkedFileData: ProjectFileAgent.decorateLinkedFileData
+	_decorateLinkedFileData: ProjectFileAgent._decorateLinkedFileData
 
 	_validate: (data) ->
 		return (
 			(data.source_project_id? || data.v1_source_doc_id?) &&
-			data.source_output_file_path?
+			data.source_output_file_path? &&
+			data.build_id?
 		)
 
-	checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) ->
+	_checkAuth: (project_id, data, current_user_id, callback = (err, allowed)->) ->
 		callback = _.once(callback)
-		if !ProjectOutputFileAgent._validate(data)
-			return callback(new BadDataError())
+		if !@_validate(data)
+			return callback(new LinkedFilesErrors.BadDataError())
 		@_getSourceProject data, (err, project) ->
 			return callback(err) if err?
-			AuthorizationManager.canUserReadProject current_user_id, project._id, null, (err, canRead) ->
-				return callback(err) if err?
-				callback(null, canRead)
+			AuthorizationManager.canUserReadProject current_user_id,
+				project._id,
+				null,
+				(err, canRead) ->
+					return callback(err) if err?
+					callback(null, canRead)
 
-	writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
+	_getFileStream: (linkedFileData, user_id, callback=(err, fileStream)->) ->
 		callback = _.once(callback)
-		if !ProjectOutputFileAgent._validate(data)
-			return callback(new BadDataError())
-		{ source_output_file_path } = data
-		@_getSourceProject data, (err, project) ->
+		{ source_output_file_path, build_id } = linkedFileData
+		@_getSourceProject linkedFileData, (err, project) ->
 			return callback(err) if err?
 			source_project_id = project._id
-			CompileManager.compile source_project_id, null, {}, (err) ->
-				return callback(err) if err?
-				ClsiManager.getOutputFileStream source_project_id, source_output_file_path, (err, readStream) ->
+			ClsiManager.getOutputFileStream source_project_id,
+				user_id,
+				build_id,
+				source_output_file_path,
+				(err, readStream) ->
 					return callback(err) if err?
 					readStream.pause()
-					readStream.on "error", callback
-					readStream.on "response", (response) ->
-						if 200 <= response.statusCode < 300
-							readStream.resume()
-							FileWriter.writeStreamToDisk project_id, readStream, callback
-						else
-							error = new OutputFileFetchFailedError("Output file fetch failed: #{url}")
-							error.statusCode = response.statusCode
-							callback(error)
+					callback(null, readStream)
 
-	handleError: (error, req, res, next) ->
-		if error instanceof ProjectFileAgent.BadDataError
-			res.status(400).send("The submitted data is not valid")
-		else if error instanceof OutputFileFetchFailedError
-			res.status(404).send("Could not get output file")
-		else if error instanceof ProjectFileAgent.ProjectNotFoundError
-			res.status(404).send("Project not found")
-		else if error instanceof ProjectFileAgent.V1ProjectNotFoundError
-			res.status(409).send(ProjectFileAgent._v1ProjectNotFoundMessage)
-		else
-			next(error)
+	_compileAndGetFileStream: (linkedFileData, user_id, callback=(err, stream, build_id)->) ->
+		callback = _.once(callback)
+		{ source_output_file_path } = linkedFileData
+		@_getSourceProject linkedFileData, (err, project) ->
+			return callback(err) if err?
+			source_project_id = project._id
+			CompileManager.compile source_project_id,
+				user_id,
+				{},
+				(err, status, outputFiles) ->
+					return callback(err) if err?
+					if status != 'success'
+						return callback(new LinkedFilesErrors.OutputFileFetchFailedError())
+					outputFile = _.find(
+						outputFiles,
+						(o) => o.path == source_output_file_path
+					)
+					if !outputFile?
+						return callback(new LinkedFilesErrors.OutputFileFetchFailedError())
+					build_id = outputFile.build
+					ClsiManager.getOutputFileStream source_project_id,
+						user_id,
+						build_id,
+						source_output_file_path,
+						(err, readStream) ->
+							return callback(err) if err?
+							readStream.pause()
+							callback(null, readStream, build_id)
 }

services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee

@@ -1,67 +1,53 @@
 request = require 'request'
-FileWriter = require('../../infrastructure/FileWriter')
 _ = require "underscore"
 urlValidator = require 'valid-url'
 Settings = require 'settings-sharelatex'
+{ InvalidUrlError, UrlFetchFailedError } = require './LinkedFilesErrors'
+LinkedFilesHandler = require './LinkedFilesHandler'
 
-UrlFetchFailedError = (message) ->
-	error = new Error(message)
-	error.name = 'UrlFetchFailedError'
-	error.__proto__ = UrlFetchFailedError.prototype
-	return error
-UrlFetchFailedError.prototype.__proto__ = Error.prototype
-
-InvalidUrlError = (message) ->
-	error = new Error(message)
-	error.name = 'InvalidUrlError'
-	error.__proto__ = InvalidUrlError.prototype
-	return error
-InvalidUrlError.prototype.__proto__ = Error.prototype
 
 module.exports = UrlAgent = {
-	UrlFetchFailedError: UrlFetchFailedError
-	InvalidUrlError: InvalidUrlError
 
-	sanitizeData: (data) ->
+	createLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+		linkedFileData = @._sanitizeData(linkedFileData)
+		@_getUrlStream project_id, linkedFileData, user_id, (err, readStream) ->
+			return callback(err) if err?
+			readStream.on "error", callback
+			readStream.on "response", (response) ->
+				if 200 <= response.statusCode < 300
+					readStream.resume()
+					LinkedFilesHandler.importFromStream project_id,
+						readStream,
+						linkedFileData,
+						name,
+						parent_folder_id,
+						user_id,
+						(err, file) ->
+							return callback(err) if err?
+							callback(null, file._id) # Created
+				else
+					error = new UrlFetchFailedError("url fetch failed: #{linkedFileData.url}")
+					error.statusCode = response.statusCode
+					callback(error)
+
+	refreshLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+		@createLinkedFile project_id, linkedFileData, name, parent_folder_id, user_id, callback
+
+	_sanitizeData: (data) ->
 		return {
+			provider: data.provider
 			url: @._prependHttpIfNeeded(data.url)
 		}
 
-	canCreate: (data) -> true
-
-	decorateLinkedFileData: (data, callback = (err, newData) ->) ->
-		return callback(null, data)
-
-	checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) ->
-		callback(null, true)
-
-	writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
+	_getUrlStream: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
 		callback = _.once(callback)
 		url = data.url
 		if !urlValidator.isWebUri(url)
 			return callback(new InvalidUrlError("invalid url: #{url}"))
-		url = UrlAgent._wrapWithProxy(url)
+		url = @_wrapWithProxy(url)
 		readStream = request.get(url)
-		readStream.on "error", callback
-		readStream.on "response", (response) ->
-			if 200 <= response.statusCode < 300
-				FileWriter.writeStreamToDisk project_id, readStream, callback
-			else
-				error = new UrlFetchFailedError("url fetch failed: #{url}")
-				error.statusCode = response.statusCode
-				callback(error)
-
-	handleError: (error, req, res, next) ->
-		if error instanceof UrlFetchFailedError
-			res.status(422).send(
-				"Your URL could not be reached (#{error.statusCode} status code). Please check it and try again."
-			)
-		else if error instanceof InvalidUrlError
-			res.status(422).send(
-				"Your URL is not valid. Please check it and try again."
-			)
-		else
-			next(error)
+		readStream.pause()
+		callback(null, readStream)
 
 	_prependHttpIfNeeded: (url) ->
 		if !url.match('://')

services/web/app/coffee/infrastructure/FileWriter.coffee

@@ -15,11 +15,14 @@ module.exports = FileWriter =
 			callback(null)
 
 	writeLinesToDisk: (identifier, lines, callback = (error, fsPath)->) ->
+		FileWriter.writeContentToDisk(identifier, lines.join('\n'), callback)
+
+	writeContentToDisk: (identifier, content, callback = (error, fsPath)->) ->
 		callback = _.once(callback)
 		fsPath = "#{Settings.path.dumpFolder}/#{identifier}_#{uuid.v4()}"
 		FileWriter._ensureDumpFolderExists (error) ->
 			return callback(error) if error?
-			fs.writeFile fsPath, lines.join('\n'), (error) ->
+			fs.writeFile fsPath, content, (error) ->
 				return callback(error) if error?
 				callback(null, fsPath)
 

services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee

@@ -218,6 +218,7 @@ define [
 				projectOutputFiles: null # or []
 				selectedProjectEntity: null
 				selectedProjectOutputFile: null
+				buildId: null
 				name: null
 			$scope.state.inFlight =
 				projects: false
@@ -349,6 +350,8 @@ define [
 						filteredFiles = resp.data.outputFiles.filter (f) ->
 							f.path.match(/.*\.(pdf|png|jpeg|jpg|gif)/)
 						$scope.data.projectOutputFiles = filteredFiles
+						$scope.data.buildId = filteredFiles?[0]?.build
+						console.log ">> build_id", $scope.data.buildId
 						_reset(err: false)
 					else
 						$scope.data.projectOutputFiles = null
@@ -368,7 +371,8 @@ define [
 					provider = 'project_output_file'
 					payload = {
 						source_project_id: projectId,
-						source_output_file_path: $scope.data.selectedProjectOutputFile
+						source_output_file_path: $scope.data.selectedProjectOutputFile,
+						build_id: $scope.data.buildId
 					}
 				else
 					provider = 'project_file'

services/web/test/acceptance/coffee/LinkedFilesTests.coffee

@@ -151,7 +151,7 @@ describe "LinkedFiles", ->
 						source_entity_path: "/#{@source_doc_name}",
 			}, (error, response, body) =>
 				expect(response.statusCode).to.equal 403
-				expect(body).to.equal 'Cannot create linked file'
+				expect(body).to.equal 'You do not have access to this project'
 				done()
 
 	describe "with a linked project_file from a v1 project that has not been imported", ->
@@ -380,6 +380,7 @@ describe "LinkedFiles", ->
 					data:
 						source_project_id: @project_two_id,
 						source_output_file_path: "output.pdf",
+						build_id: '1234-abcd'
 			}, (error, response, body) =>
 				new_file_id = body.new_file_id
 				@existing_file_id = new_file_id
@@ -393,6 +394,7 @@ describe "LinkedFiles", ->
 						source_project_id: @project_two_id,
 						source_output_file_path: "output.pdf",
 						source_project_display_name: "output-test-two"
+						build_id: '1234-abcd'
 					}
 					expect(firstFile.name).to.equal('test.pdf')
 					done()

services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee

@@ -33,12 +33,26 @@ module.exports = MockClsiApi =
 
 		app.post "/project/:project_id/compile", (req, res, next) =>
 			res.json {
-				outputFiles: [{path: 'output.pdf'}]
+				compile:
+					status: 'success'
+					outputFiles: [{path: 'output.pdf', build: 'abcd', url: 'http://example.com'}]
+			}
+		app.post "/project/:project_id/user/:user_id/compile", (req, res, next) =>
+			res.json {
+				compile:
+					status: 'success'
+					outputFiles: [{path: 'output.pdf', build: 'abcd', url: 'http://example.com'}]
 			}
 
-		app.get "/project/:project_id/output/:output_path", (req, res, next) =>
+		app.get "/project/:project_id/status", (req, res, next) =>
+			res.status(200).send()
+
+		app.get "/project/:project_id/user/:user_id/build/:build_id/output/:output_path", (req, res, next) =>
 			res.status(200).send("hello")
 
+		app.all "*", (req, res, next) =>
+			next()
+
 		app.listen 3013, (error) ->
 			throw error if error?
 		.on "error", (error) ->