Merge pull request #335 from sharelatex/ja-linked-url-files

Add in support for URL based linked files

services/web/app/coffee/Features/Editor/EditorController.coffee

@@ -23,15 +23,15 @@ module.exports = EditorController =
 			EditorRealTimeController.emitToRoom(project_id, 'reciveNewDoc', folder_id, doc, source)
 			callback(err, doc)
 
-	addFile: (project_id, folder_id, fileName, fsPath, source, user_id, callback = (error, file)->)->
+	addFile: (project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id, callback = (error, file)->)->
 		fileName = fileName.trim()
-		logger.log {project_id, folder_id, fileName, fsPath}, "sending new file to project"
+		logger.log {project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id}, "sending new file to project"
 		Metrics.inc "editor.add-file"
-		ProjectEntityUpdateHandler.addFile project_id, folder_id, fileName, fsPath, user_id, (err, fileRef, folder_id)=>
+		ProjectEntityUpdateHandler.addFile project_id, folder_id, fileName, fsPath, linkedFileData, user_id, (err, fileRef, folder_id)=>
 			if err?
 				logger.err err:err, project_id:project_id, folder_id:folder_id, fileName:fileName, "error adding file without lock"
 				return callback(err)
-			EditorRealTimeController.emitToRoom(project_id, 'reciveNewFile', folder_id, fileRef, source)
+			EditorRealTimeController.emitToRoom(project_id, 'reciveNewFile', folder_id, fileRef, source, linkedFileData)
 			callback(err, fileRef)
 
 	upsertDoc: (project_id, folder_id, docName, docLines, source, user_id, callback = (err)->)->
@@ -40,11 +40,11 @@ module.exports = EditorController =
 				EditorRealTimeController.emitToRoom(project_id, 'reciveNewDoc', folder_id, doc, source)
 			callback err, doc
 
-	upsertFile: (project_id, folder_id, fileName, fsPath, source, user_id, callback = (err, file) ->) ->
-		ProjectEntityUpdateHandler.upsertFile project_id, folder_id, fileName, fsPath, user_id, (err, file, didAddFile) ->
+	upsertFile: (project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id, callback = (err, file) ->) ->
+		ProjectEntityUpdateHandler.upsertFile project_id, folder_id, fileName, fsPath, linkedFileData, user_id, (err, file, didAddFile) ->
 			return callback(err) if err?
 			if didAddFile
-				EditorRealTimeController.emitToRoom project_id, 'reciveNewFile', folder_id, file, source
+				EditorRealTimeController.emitToRoom project_id, 'reciveNewFile', folder_id, file, source, linkedFileData
 			callback null, file
 
 	upsertDocWithPath: (project_id, elementPath, docLines, source, user_id, callback) ->
@@ -56,13 +56,13 @@ module.exports = EditorController =
 					EditorRealTimeController.emitToRoom project_id, 'reciveNewDoc', lastFolder._id, doc, source
 				callback()
 
-	upsertFileWithPath: (project_id, elementPath, fsPath, source, user_id, callback) ->
-		ProjectEntityUpdateHandler.upsertFileWithPath project_id, elementPath, fsPath, user_id, (err, file, didAddFile, newFolders, lastFolder) ->
+	upsertFileWithPath: (project_id, elementPath, fsPath, linkedFileData, source, user_id, callback) ->
+		ProjectEntityUpdateHandler.upsertFileWithPath project_id, elementPath, fsPath, linkedFileData, user_id, (err, file, didAddFile, newFolders, lastFolder) ->
 			return callback(err) if err?
 			EditorController._notifyProjectUsersOfNewFolders project_id, newFolders, (err) ->
 				return callback(err) if err?
 				if didAddFile
-					EditorRealTimeController.emitToRoom project_id, 'reciveNewFile', lastFolder._id, file, source
+					EditorRealTimeController.emitToRoom project_id, 'reciveNewFile', lastFolder._id, file, source, linkedFileData
 				callback()
 
 	addFolder : (project_id, folder_id, folderName, source, callback = (error, folder)->)->

services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee

@@ -0,0 +1,32 @@
+AuthenticationController = require '../Authentication/AuthenticationController'
+EditorController = require '../Editor/EditorController'
+Settings = require 'settings-sharelatex'
+logger = require 'logger-sharelatex'
+
+module.exports = LinkedFilesController = {
+	Agents: {
+		url: require('./UrlAgent')
+	}
+
+	createLinkedFile: (req, res, next) ->
+		{project_id} = req.params
+		{name, provider, data, parent_folder_id} = req.body
+		user_id = AuthenticationController.getLoggedInUserId(req)
+		logger.log {project_id, name, provider, data, parent_folder_id, user_id}, 'create linked file request'
+
+		if !LinkedFilesController.Agents.hasOwnProperty(provider)
+			return res.send(400)
+		unless provider in Settings.enabledLinkedFileTypes
+			return res.send(400)
+		Agent = LinkedFilesController.Agents[provider]
+
+		linkedFileData = Agent.sanitizeData(data)
+		linkedFileData.provider = provider
+		Agent.writeIncomingFileToDisk project_id, linkedFileData, user_id, (error, fsPath) ->
+			if error?
+				logger.error {err: error, project_id, name, linkedFileData, parent_folder_id, user_id}, 'error writing linked file to disk'
+				return Agent.handleError(error, req, res, next)
+			EditorController.upsertFile project_id, parent_folder_id, name, fsPath, linkedFileData, "upload", user_id, (error) ->
+				return next(error) if error?
+				res.send(204) # created
+}

services/web/app/coffee/Features/LinkedFiles/LinkedFilesRouter.coffee

@@ -0,0 +1,11 @@
+AuthorizationMiddlewear = require('../Authorization/AuthorizationMiddlewear')
+AuthenticationController = require('../Authentication/AuthenticationController')
+LinkedFilesController = require "./LinkedFilesController"
+
+module.exports =
+	apply: (webRouter) ->
+		webRouter.post '/project/:project_id/linked_file',
+			AuthenticationController.requireLogin(),
+			AuthorizationMiddlewear.ensureUserCanWriteProjectContent,
+			LinkedFilesController.createLinkedFile
+

services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee

@@ -0,0 +1,68 @@
+request = require 'request'
+FileWriter = require('../../infrastructure/FileWriter')
+_ = require "underscore"
+urlValidator = require 'valid-url'
+Settings = require 'settings-sharelatex'
+
+UrlFetchFailedError = (message) ->
+	error = new Error(message)
+	error.name = 'UrlFetchFailedError'
+	error.__proto__ = UrlFetchFailedError.prototype
+	return error
+UrlFetchFailedError.prototype.__proto__ = Error.prototype
+
+InvalidUrlError = (message) ->
+	error = new Error(message)
+	error.name = 'InvalidUrlError'
+	error.__proto__ = InvalidUrlError.prototype
+	return error
+InvalidUrlError.prototype.__proto__ = Error.prototype
+
+module.exports = UrlAgent = {
+	UrlFetchFailedError: UrlFetchFailedError
+	InvalidUrlError: InvalidUrlError
+
+	sanitizeData: (data) ->
+		return {
+			url: @._prependHttpIfNeeded(data.url)
+		}
+
+	writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
+		callback = _.once(callback)
+		url = data.url
+		if !urlValidator.isWebUri(url)
+			return callback(new InvalidUrlError("invalid url: #{url}"))
+		url = UrlAgent._wrapWithProxy(url)
+		readStream = request.get(url)
+		readStream.on "error", callback
+		readStream.on "response", (response) ->
+			if 200 <= response.statusCode < 300
+				FileWriter.writeStreamToDisk project_id, readStream, callback
+			else
+				error = new UrlFetchFailedError("url fetch failed: #{url}")
+				error.statusCode = response.statusCode
+				callback(error)
+
+	handleError: (error, req, res, next) ->
+		if error instanceof UrlFetchFailedError
+			res.status(422).send(
+				"Your URL could not be reached (#{error.statusCode} status code). Please check it and try again."
+			)
+		else if error instanceof InvalidUrlError
+			res.status(422).send(
+				"Your URL is not valid. Please check it and try again."
+			)
+		else
+			next(error)
+
+	_prependHttpIfNeeded: (url) ->
+		if !url.match('://')
+			url = 'http://' + url
+		return url
+
+	_wrapWithProxy: (url) ->
+		# TODO: Consider what to do for Community and Enterprise edition?
+		if !Settings.apis?.linkedUrlProxy?.url?
+			throw new Error('no linked url proxy configured')
+		return "#{Settings.apis.linkedUrlProxy.url}?url=#{encodeURIComponent(url)}"
+}

services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee

@@ -79,7 +79,7 @@ module.exports = ProjectCreationHandler =
 							callback(error)
 				(callback) ->
 					universePath = Path.resolve(__dirname + "/../../../templates/project_files/universe.jpg")
-					ProjectEntityUpdateHandler.addFile project._id, project.rootFolder[0]._id, "universe.jpg", universePath, owner_id, callback
+					ProjectEntityUpdateHandler.addFile project._id, project.rootFolder[0]._id, "universe.jpg", universePath, null, owner_id, callback
 			], (error) ->
 				callback(error, project)
 

services/web/app/coffee/Features/Project/ProjectEditorHandler.coffee

@@ -76,6 +76,8 @@ module.exports = ProjectEditorHandler =
 	buildFileModelView: (file) ->
 		_id  : file._id
 		name : file.name
+		linkedFileData: file.linkedFileData
+		created: file.created
 
 	buildDocModelView: (doc) ->
 		_id   : doc._id

services/web/app/coffee/Features/Project/ProjectEntityMongoUpdateHandler.coffee

@@ -48,7 +48,7 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
 			self._confirmFolder project, folder_id, (folder_id)->
 				self._putElement project, folder_id, fileRef, "file", callback
 
-	replaceFile: wrapWithLock (project_id, file_id, callback) ->
+	replaceFile: wrapWithLock (project_id, file_id, linkedFileData, callback) ->
 		ProjectGetter.getProjectWithoutLock project_id, {rootFolder: true, name:true}, (err, project) ->
 			return callback(err) if err?
 			ProjectLocator.findElement {project:project, element_id: file_id, type: 'file'}, (err, fileRef, path)=>
@@ -63,6 +63,7 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
 				inc['version'] = 1
 				set = {}
 				set["#{path.mongo}.created"] = new Date()
+				set["#{path.mongo}.linkedFileData"] = linkedFileData
 				update =
 					"$inc": inc
 					"$set": set

services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee

@@ -129,8 +129,8 @@ module.exports = ProjectEntityUpdateHandler = self =
 				return callback(error) if error?
 				callback null, doc, folder_id
 
-	addFile: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (error, fileRef, folder_id) ->)->
-		self.addFileWithoutUpdatingHistory.withoutLock project_id, folder_id, fileName, fsPath, userId, (error, fileRef, folder_id, path, fileStoreUrl) ->
+	addFile: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (error, fileRef, folder_id) ->)->
+		self.addFileWithoutUpdatingHistory.withoutLock project_id, folder_id, fileName, fsPath, linkedFileData, userId, (error, fileRef, folder_id, path, fileStoreUrl) ->
 			return callback(error) if error?
 			newFiles = [
 				file: fileRef
@@ -141,10 +141,10 @@ module.exports = ProjectEntityUpdateHandler = self =
 				return callback(error) if error?
 				callback null, fileRef, folder_id
 
-	replaceFile: wrapWithLock (project_id, file_id, fsPath, userId, callback)->
+	replaceFile: wrapWithLock (project_id, file_id, fsPath, linkedFileData, userId, callback)->
 		FileStoreHandler.uploadFileFromDisk project_id, file_id, fsPath, (err, fileStoreUrl)->
 			return callback(err) if err?
-			ProjectEntityMongoUpdateHandler.replaceFile project_id, file_id, (err, fileRef, project, path) ->
+			ProjectEntityMongoUpdateHandler.replaceFile project_id, file_id, linkedFileData, (err, fileRef, project, path) ->
 				return callback(err) if err?
 				newFiles = [
 					file: fileRef
@@ -180,7 +180,7 @@ module.exports = ProjectEntityUpdateHandler = self =
 					return callback(err) if err?
 					callback(null, doc, folder_id, result?.path?.fileSystem)
 
-	addFileWithoutUpdatingHistory: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (error, fileRef, folder_id, path, fileStoreUrl) ->)->
+	addFileWithoutUpdatingHistory: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (error, fileRef, folder_id, path, fileStoreUrl) ->)->
 		# This method should never be called directly, except when importing a project
 		# from Overleaf. It skips sending updates to the project history, which will break
 		# the history unless you are making sure it is updated in some other way.
@@ -188,7 +188,10 @@ module.exports = ProjectEntityUpdateHandler = self =
 		if not SafePath.isCleanFilename fileName
 			return callback new Errors.InvalidNameError("invalid element name")
 
-		fileRef = new File name : fileName
+		fileRef = new File(
+			name: fileName
+			linkedFileData: linkedFileData
+		)
 		FileStoreHandler.uploadFileFromDisk project_id, fileRef._id, fsPath, (err, fileStoreUrl)->
 			if err?
 				logger.err err:err, project_id: project_id, folder_id: folder_id, file_name: fileName, fileRef:fileRef, "error uploading image to s3"
@@ -221,7 +224,7 @@ module.exports = ProjectEntityUpdateHandler = self =
 					return callback(err) if err?
 					callback null, doc, !existingDoc?
 
-	upsertFile: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (err, file, isNewFile)->)->
+	upsertFile: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (err, file, isNewFile)->)->
 		ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
 			return callback(error) if error?
 			return callback(new Error("Couldn't find folder")) if !folder?
@@ -231,11 +234,11 @@ module.exports = ProjectEntityUpdateHandler = self =
 					existingFile = fileRef
 					break
 			if existingFile?
-				self.replaceFile.withoutLock project_id, existingFile._id, fsPath, userId, (err) ->
+				self.replaceFile.withoutLock project_id, existingFile._id, fsPath, linkedFileData, userId, (err) ->
 					return callback(err) if err?
 					callback null, existingFile, !existingFile?
 			else
-				self.addFile.withoutLock project_id, folder_id, fileName, fsPath, userId, (err, file) ->
+				self.addFile.withoutLock project_id, folder_id, fileName, fsPath, linkedFileData, userId, (err, file) ->
 					return callback(err) if err?
 					callback null, file, !existingFile?
 
@@ -248,12 +251,12 @@ module.exports = ProjectEntityUpdateHandler = self =
 				return callback(err) if err?
 				callback null, doc, isNewDoc, newFolders, folder
 
-	upsertFileWithPath: wrapWithLock (project_id, elementPath, fsPath, userId, callback) ->
+	upsertFileWithPath: wrapWithLock (project_id, elementPath, fsPath, linkedFileData, userId, callback) ->
 		fileName = path.basename(elementPath)
 		folderPath = path.dirname(elementPath)
 		self.mkdirp.withoutLock project_id, folderPath, (err, newFolders, folder) ->
 			return callback(err) if err?
-			self.upsertFile.withoutLock project_id, folder._id, fileName, fsPath, userId, (err, file, isNewFile) ->
+			self.upsertFile.withoutLock project_id, folder._id, fileName, fsPath, linkedFileData, userId, (err, file, isNewFile) ->
 				return callback(err) if err?
 				callback null, file, isNewFile, newFolders, folder
 

services/web/app/coffee/Features/ThirdPartyDataStore/UpdateMerger.coffee

@@ -1,15 +1,14 @@
 _ = require('underscore')
 fs = require('fs')
 logger = require('logger-sharelatex')
-uuid = require('uuid')
 EditorController = require('../Editor/EditorController')
 FileTypeManager = require('../Uploads/FileTypeManager')
-Settings = require('settings-sharelatex')
+FileWriter = require('../../infrastructure/FileWriter')
 
 module.exports = UpdateMerger =
 	mergeUpdate: (user_id, project_id, path, updateRequest, source, callback = (error) ->)->
 		logger.log project_id:project_id, path:path, "merging update from tpds"
-		UpdateMerger.p.writeStreamToDisk project_id, updateRequest, (err, fsPath)->
+		FileWriter.writeStreamToDisk project_id, updateRequest, (err, fsPath)->
 			return callback(err) if err?
 			UpdateMerger._mergeUpdate user_id, project_id, path, fsPath, source, (mergeErr) ->
 				fs.unlink fsPath, (deleteErr) ->
@@ -44,29 +43,10 @@ module.exports = UpdateMerger =
 
 		processFile: (project_id, fsPath, path, source, user_id, callback)->
 			logger.log project_id:project_id, "processing file update from tpds"
-			EditorController.upsertFileWithPath project_id, path, fsPath, source, user_id, (err) ->
+			EditorController.upsertFileWithPath project_id, path, fsPath, null, source, user_id, (err) ->
 				logger.log project_id:project_id, "completed processing file update from tpds"
 				callback(err)
 
-		writeStreamToDisk: (project_id, stream, callback = (err, fsPath)->)->
-			dumpPath = "#{Settings.path.dumpFolder}/#{project_id}_#{uuid.v4()}"
-
-			writeStream = fs.createWriteStream(dumpPath)
-			stream.pipe(writeStream)
-
-			stream.on 'error', (err)->
-				logger.err {err, project_id, dumpPath},
-					"something went wrong with incoming tpds update stream"
-			writeStream.on 'error', (err)->
-				logger.err {err, project_id, dumpPath},
-					"something went wrong with writing tpds update to disk"
-
-			stream.on 'end', ->
-				logger.log {project_id, dumpPath}, "incoming tpds update stream ended"
-			writeStream.on "finish", ->
-				logger.log {project_id, dumpPath}, "tpds update write stream finished"
-				callback null, dumpPath
-
 		readFileIntoTextArray: (path, callback)->
 			fs.readFile path, "utf8", (error, content = "") ->
 				if error?

services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee

@@ -27,9 +27,9 @@ module.exports = FileSystemImportManager =
 				return callback("path is symlink")
 
 			if replace
-				EditorController.upsertFile project_id, folder_id, name, path, "upload", user_id, callback
+				EditorController.upsertFile project_id, folder_id, name, path, null, "upload", user_id, callback
 			else
-				EditorController.addFile project_id, folder_id, name, path, "upload", user_id, callback
+				EditorController.addFile project_id, folder_id, name, path, null, "upload", user_id, callback
 
 	addFolder: (user_id, project_id, folder_id, name, path, replace, callback = (error)-> ) ->
 		FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->

services/web/app/coffee/infrastructure/FileWriter.coffee

@@ -0,0 +1,23 @@
+fs = require 'fs'
+logger = require 'logger-sharelatex'
+uuid = require 'uuid'
+_ = require 'underscore'
+Settings = require 'settings-sharelatex'
+
+module.exports = 
+	writeStreamToDisk: (identifier, stream, callback = (error, fsPath) ->) ->
+		callback = _.once(callback)
+		fsPath = "#{Settings.path.dumpFolder}/#{identifier}_#{uuid.v4()}"
+
+		writeStream = fs.createWriteStream(fsPath)
+		stream.pipe(writeStream)
+
+		stream.on 'error', (err)->
+			logger.err {err, identifier, fsPath},	"[writeStreamToDisk] something went wrong with incoming stream"
+			callback(err)
+		writeStream.on 'error', (err)->
+			logger.err {err, identifier, fsPath},	"[writeStreamToDisk] something went wrong with writing to disk"
+			callback(err)
+		writeStream.on "finish", ->
+			logger.log {identifier, fsPath}, "[writeStreamToDisk] write stream finished"
+			callback null, fsPath

services/web/app/coffee/models/File.coffee

@@ -8,6 +8,7 @@ FileSchema = new Schema
 	name            :     type:String, default:''
 	created     	:     type:Date, default: () -> new Date()
 	rev 			:	  {type:Number, default:0}
+	linkedFileData: { type: Schema.Types.Mixed }
 
 mongoose.model 'File', FileSchema
 exports.File = mongoose.model 'File'

services/web/app/coffee/router.coffee

@@ -46,6 +46,7 @@ AnnouncementsController = require("./Features/Announcements/AnnouncementsControl
 MetaController = require('./Features/Metadata/MetaController')
 TokenAccessController = require('./Features/TokenAccess/TokenAccessController')
 Features = require('./infrastructure/Features')
+LinkedFilesRouter = require './Features/LinkedFiles/LinkedFilesRouter'
 
 logger = require("logger-sharelatex")
 _ = require("underscore")
@@ -77,6 +78,7 @@ module.exports = class Router
 		RealTimeProxyRouter.apply(webRouter, privateApiRouter)
 		ContactRouter.apply(webRouter, privateApiRouter)
 		AnalyticsRouter.apply(webRouter, privateApiRouter, publicApiRouter)
+		LinkedFilesRouter.apply(webRouter, privateApiRouter, publicApiRouter)
 
 		Modules.applyRouter(webRouter, privateApiRouter, publicApiRouter)
 

services/web/app/views/project/editor.pug

@@ -106,7 +106,7 @@ block requirejs
 	//- We need to do .replace(/\//g, '\\/') do that '</script>' -> '<\/script>'
 	//- and doesn't prematurely end the script tag.
 	script#data(type="application/json").
-		!{JSON.stringify({userSettings: userSettings, user: user, trackChangesState: trackChangesState, useV2History: useV2History}).replace(/\//g, '\\/')}
+		!{JSON.stringify({userSettings: userSettings, user: user, trackChangesState: trackChangesState, useV2History: useV2History, enabledLinkedFileTypes: settings.enabledLinkedFileTypes}).replace(/\//g, '\\/')}
 
 	script(type="text/javascript").
 		window.data = JSON.parse($("#data").text());
@@ -115,6 +115,7 @@ block requirejs
 		var data = JSON.parse($("#data").text());
 		window.userSettings = data.userSettings;
 		window.user = data.user;
+		window.enabledLinkedFiles = data.enabledLinkedFiles;
 		window.csrfToken = "!{csrfToken}";
 		window.anonymous = #{anonymous};
 		window.anonymousAccessToken = "#{anonymousAccessToken}";

services/web/app/views/project/editor/binary-file.pug

@@ -6,7 +6,7 @@ div.binary-file.full-size(
 	img(
 		ng-show="!failedLoad"
 		ng-src="/project/{{ project_id }}/file/{{ openFile.id }}"
-		ng-if="['png', 'jpg', 'jpeg', 'gif'].indexOf(extension(openFile)) > -1"
+		ng-if="isImageFile()"
 		ng-class="{'img-preview': !imgLoaded}"
 		onerror="sl_binaryFilePreviewError()"
 		onabort="sl_binaryFilePreviewError()"
@@ -16,29 +16,50 @@ div.binary-file.full-size(
 	img(
 		ng-show="!failedLoad"
 		ng-src="/project/{{ project_id }}/file/{{ openFile.id }}?format=png"
-		ng-if="['pdf', 'eps'].indexOf(extension(openFile)) > -1"
+		ng-if="isPreviewableFile()"
 		ng-class="{'img-preview': !imgLoaded}"
 		onerror="sl_binaryFilePreviewError()"
 		onabort="sl_binaryFilePreviewError()"
 		onload="sl_binaryFilePreviewLoaded()"
 	)
 	
-	div(ng-if="(['bib'].indexOf(extension(openFile)) > -1) && !bibtexPreview.error")
-
-		div.bib-loading(ng-show="bibtexPreview.loading && !bibtexPreview.error")
+	div(ng-if="isTextFile() && !textPreview.error")
+		div.text-loading(ng-show="textPreview.loading && !textPreview.error")
 			| #{translate('loading')}...
-
-		div.bib-preview(ng-show="bibtexPreview.data && !bibtexPreview.loading && !bibtexPreview.error")
+		div.text-preview(ng-show="textPreview.data && !textPreview.loading && !textPreview.error")
 			div.scroll-container
 				p
-					| {{ bibtexPreview.data }}
-				p(ng-show="bibtexPreview.shouldShowDots")
+					| {{ textPreview.data }}
+				p(ng-show="textPreview.shouldShowDots")
 					| ...
 	
 	p.no-preview(
-		ng-if="failedLoad || bibtexPreview.error || ['bib', 'png', 'jpg', 'jpeg', 'gif', 'pdf', 'eps'].indexOf(extension(openFile)) == -1"
-	) #{translate("no_preview_available")}
+		ng-if="failedLoad || textPreview.error || isUnpreviewableFile()"
+	) #{translate("no_preview_available")} {{ failedLoad }} {{ textPreview.error }} {{ isUnpreviewableFile() }}
+
+	div.binary-file-footer
+		div(ng-show="openFile.linkedFileData.provider == 'url'")
+			p
+				i.fa.fa-fw.fa-external-link-square.fa-rotate-180.linked-file-icon
+				| Imported from
+				|
+				a(ng-href='{{openFile.linkedFileData.url}}') {{ displayUrl(openFile.linkedFileData.url) }}
+				|
+				| at {{ openFile.created | formatDate:'h:mm a' }} {{ openFile.created | relativeDate }}
 
-	a.btn.btn-info(
-		ng-href="/project/{{ project_id }}/file/{{ openFile.id }}"
-	) #{translate("download")} {{ openFile.name }}
+		span(ng-show="openFile.linkedFileData.provider == 'url'")
+			button.btn.btn-success(
+				href, ng-click="refreshFile(openFile)",
+				ng-disabled="refreshing"
+			)
+				i.fa.fa-fw.fa-refresh(ng-class={'fa-spin': refreshing})
+				|
+				span(ng-show="!refreshing") Refresh
+				span(ng-show="refreshing") Refreshing...
+			|  
+		a.btn.btn-info(
+			ng-href="/project/{{ project_id }}/file/{{ openFile.id }}"
+		)
+			i.fa.fa-fw.fa-download
+			|
+			| #{translate("download")}

services/web/app/views/project/editor/file-tree.pug

@@ -108,6 +108,9 @@ script(type='text/ng-template', id='entityListItemTemplate')
 				i.fa.fa-fw.toggle(ng-if="entity.type != 'folder'")
 
 				i.fa.fa-fw(ng-if="entity.type != 'folder'", ng-class="'fa-' + iconTypeFromName(entity.name)")
+					i.fa.fa-external-link-square.fa-rotate-180.linked-file-highlight(
+						ng-if="entity.linkedFileData.provider"
+					)
 				span(
 					ng-hide="entity.renaming"
 				)  {{ entity.renamingToName || entity.name }}
@@ -339,6 +342,52 @@ script(type='text/ng-template', id='newDocModalTemplate')
 			span(ng-show="state.inflight") #{translate("creating")}...
 
 
+script(type='text/ng-template', id='linkedFileModalTemplate')
+	.modal-header
+		h3 New file from URL
+	.modal-body
+		form(novalidate, name="newLinkedFileForm")
+			div.alert.alert-danger(ng-if="error")
+				div(ng-switch="error")
+					span(ng-switch-when="already exists") #{translate("file_already_exists")}
+					span(ng-switch-default) {{error}}
+			label(for="url") URL to fetch the file from
+			input.form-control(
+				type="text",
+				placeholder="www.example.com/my_file",
+				required,
+				ng-model="inputs.url",
+				focus-on="open",
+				on-enter="create()",
+				name="url"
+			)
+			.row-spaced
+				label(for="name") File name in this project
+				input.form-control(
+					type="text",
+					placeholder="my_file",
+					required,
+					ng-model="inputs.name",
+					ng-change="nameChangedByUser = true"
+					valid-file,
+					on-enter="create()",
+					name="name"
+				)
+				.text-danger.row-spaced-small(ng-show="newDocForm.name.$error.validFile")
+					| #{translate('files_cannot_include_invalid_characters')}
+	.modal-footer
+		button.btn.btn-default(
+			ng-disabled="state.inflight"
+			ng-click="cancel()"
+		) #{translate("cancel")}
+		button.btn.btn-primary(
+			ng-disabled="newLinkedFileForm.$invalid || state.inflight"
+			ng-click="create()"
+		)
+			span(ng-hide="state.inflight") #{translate("create")}
+			span(ng-show="state.inflight") #{translate("creating")}...
+
+
 script(type='text/ng-template', id='newFolderModalTemplate')
 	.modal-header
 		h3 #{translate("new_folder")}

services/web/config/settings.defaults.coffee

@@ -152,6 +152,8 @@ module.exports = settings =
 			url: "http://#{process.env['NOTIFICATIONS_HOST'] or 'localhost'}:3042"
 		analytics:
 			url: "http://#{process.env['ANALYTICS_HOST'] or 'localhost'}:3050"
+		linkedUrlProxy:
+			url: process.env['LINKED_URL_PROXY']
 
 	templates:
 		user_id: process.env.TEMPLATES_USER_ID or "5395eb7aad1f29a88756c7f2"
@@ -214,6 +216,8 @@ module.exports = settings =
 
 	enableSubscriptions:false
 
+	enabledLinkedFileTypes: (process.env['ENABLED_LINKED_FILE_TYPES'] or '').split(',')
+
 	# i18n
 	# ------
 	#

services/web/docker-compose.yml

@@ -15,6 +15,8 @@ services:
       MONGO_URL: "mongodb://mongo/sharelatex"
       SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true'
       PROJECT_HISTORY_ENABLED: 'true'
+      ENABLED_LINKED_FILE_TYPES: 'url'
+      LINKED_URL_PROXY: 'http://localhost:6543'
     depends_on:
       - redis
       - mongo

services/web/nodemon.frontend.json

@@ -4,6 +4,7 @@
     "node_modules/"
   ],
   "verbose": true,
+  "legacyWatch": true,
   "exec": "make compile",
   "watch": [
     "public/coffee/",

services/web/nodemon.json

@@ -4,6 +4,7 @@
     "node_modules/"
   ],
   "verbose": true,
+  "legacyWatch": true,
   "execMap": {
     "js": "npm run start"
   },

services/web/package.json

@@ -85,6 +85,7 @@
     "underscore": "1.6.0",
     "uuid": "^3.0.1",
     "v8-profiler": "^5.2.3",
+    "valid-url": "^1.0.9",
     "xml2js": "0.2.0",
     "yauzl": "^2.8.0"
   },

services/web/public/coffee/ide/binary-files/controllers/BinaryFileController.coffee

@@ -1,16 +1,57 @@
 define [
 	"base"
-], (App) ->
-	App.controller "BinaryFileController", ["$scope", "$rootScope", "$http", "$timeout", ($scope, $rootScope, $http, $timeout) ->
+	"moment"
+], (App, moment) ->
+	App.controller "BinaryFileController", ["$scope", "$rootScope", "$http", "$timeout", "$element", "ide", ($scope, $rootScope, $http, $timeout, $element, ide) ->
 
 		TWO_MEGABYTES = 2 * 1024 * 1024
 
-		$scope.bibtexPreview =
+		textExtensions = ['bib', 'tex', 'txt', 'cls', 'sty']
+		imageExtentions = ['png', 'jpg', 'jpeg', 'gif']
+		previewableExtensions = ['eps', 'pdf']
+
+		extension = (file) ->
+			return file.name.split(".").pop()?.toLowerCase()
+
+		$scope.isTextFile = () =>
+			textExtensions.indexOf(extension($scope.openFile)) > -1
+		$scope.isImageFile = () =>
+			imageExtentions.indexOf(extension($scope.openFile)) > -1
+		$scope.isPreviewableFile = () =>
+			previewableExtensions.indexOf(extension($scope.openFile)) > -1
+		$scope.isUnpreviewableFile = () ->
+			!$scope.isTextFile() and
+			!$scope.isImageFile() and
+			!$scope.isPreviewableFile()
+
+		$scope.textPreview =
 			loading: false
 			shouldShowDots: false
 			error: false
 			data: null
 
+		$scope.refreshing = false
+
+		MAX_URL_LENGTH = 60
+		FRONT_OF_URL_LENGTH = 35
+		FILLER = '...'
+		TAIL_OF_URL_LENGTH = MAX_URL_LENGTH - FRONT_OF_URL_LENGTH - FILLER.length
+		$scope.displayUrl = (url) ->
+			if url.length > MAX_URL_LENGTH
+				front = url.slice(0, FRONT_OF_URL_LENGTH)
+				tail = url.slice(url.length - TAIL_OF_URL_LENGTH)
+				return front + FILLER + tail
+			else
+				return url
+
+		$scope.refreshFile = (file) ->
+			$scope.refreshing = true
+			ide.fileTreeManager.refreshLinkedFile(file)
+				.then () ->
+					loadTextFileFilePreview()
+				.finally () ->
+					$scope.refreshing = false
+
 		# Callback fired when the `img` tag fails to load,
 		# `failedLoad` used to show the "No Preview" message
 		$scope.failedLoad = false
@@ -25,47 +66,41 @@ define [
 			$scope.imgLoaded = true
 			$scope.$apply()
 
-		$scope.extension = (file) ->
-			return file.name.split(".").pop()?.toLowerCase()
-
-		$scope.loadBibtexFilePreview = () ->
+		do loadTextFileFilePreview = () ->
+			return unless $scope.isTextFile()
 			url = "/project/#{project_id}/file/#{$scope.openFile.id}?range=0-#{TWO_MEGABYTES}"
-			$scope.bibtexPreview.loading = true
-			$scope.bibtexPreview.shouldShowDots = false
+			$scope.textPreview.data = null
+			$scope.textPreview.loading = true
+			$scope.textPreview.shouldShowDots = false
 			$scope.$apply()
-			$http.get(url)
+			$http({
+				url: url,
+				method: 'GET',
+				transformResponse: null # Don't parse JSON
+			})
 				.then (response) ->
 					{ data } = response
-					$scope.bibtexPreview.loading = false
-					$scope.bibtexPreview.error = false
+					$scope.textPreview.error = false
 					# show dots when payload is closs to cutoff
 					if data.length >= (TWO_MEGABYTES - 200)
-						$scope.bibtexPreview.shouldShowDots = true
+						$scope.textPreview.shouldShowDots = true
 					try
 						# remove last partial line
 						data = data.replace(/\n.*$/, '')
 					finally
-						$scope.bibtexPreview.data = data
-					$timeout($scope.setHeight, 0)
-				.catch () ->
-					$scope.bibtexPreview.error = true
-					$scope.bibtexPreview.loading = false
-
-		$scope.setHeight = () ->
-			# Behold, a ghastly hack
-			guide = document.querySelector('.file-tree-inner')
-			table_wrap = document.querySelector('.bib-preview .scroll-container')
-			if table_wrap
-				desired_height = guide.offsetHeight - 44
-				if table_wrap.offsetHeight > desired_height
-					table_wrap.style.height = desired_height + 'px'
-					table_wrap.style['max-height'] = desired_height + 'px'
-
-		$scope.loadBibtexIfRequired = () ->
-			if $scope.extension($scope.openFile) == 'bib'
-				$scope.bibtexPreview.data = null
-				$scope.loadBibtexFilePreview()
+						$scope.textPreview.data = data
+					$timeout(setHeight, 0)
+				.catch (error) ->
+					console.error(error)
+					$scope.textPreview.error = true
+					$scope.textPreview.loading = false
 
-		$scope.loadBibtexIfRequired()
+		setHeight = () ->
+			$preview = $element.find('.text-preview .scroll-container')
+			$footer = $element.find('.binary-file-footer')
+			maxHeight = $element.height() - $footer.height() - 14 # borders + margin
+			$preview.css('max-height': maxHeight)
+			# Don't show the preview until we've set the height, otherwise we jump around
+			$scope.textPreview.loading = false
 
 	]

services/web/public/coffee/ide/file-tree/FileTreeManager.coffee

@@ -37,13 +37,14 @@ define [
 					}
 					@recalculateDocList()
 
-			@ide.socket.on "reciveNewFile", (parent_folder_id, file) =>
+			@ide.socket.on "reciveNewFile", (parent_folder_id, file, source, linkedFileData) =>
 				parent_folder = @findEntityById(parent_folder_id) or @$scope.rootFolder
 				@$scope.$apply () =>
 					parent_folder.children.push {
 						name: file.name
 						id:   file._id
-						type: "file"
+						type: "file",
+						linkedFileData: linkedFileData
 					}
 					@recalculateDocList()
 
@@ -175,6 +176,9 @@ define [
 		_findEntityByPathInFolder: (folder, path) ->
 			if !path? or !folder?
 				return null
+			if path == ""
+				return folder
+
 			parts = path.split("/")
 			name = parts.shift()
 			rest = parts.join("/")
@@ -222,10 +226,19 @@ define [
 		getRootDocDirname: () ->
 			rootDoc = @findEntityById @$scope.project.rootDoc_id
 			return if !rootDoc?
-			path = @getEntityPath(rootDoc)
+			return @_getEntityDirname(rootDoc)
+
+		_getEntityDirname: (entity) ->
+			path = @getEntityPath(entity)
 			return if !path?
 			return path.split("/").slice(0, -1).join("/")
 
+		_findParentFolder: (entity) ->
+			dirname = @_getEntityDirname(entity)
+			console.log('dirname', dirname)
+			return if !dirname?
+			return @findEntityByPath(dirname)
+
 		loadRootFolder: () ->
 			@$scope.rootFolder = @_parseFolder(@$scope?.project?.rootFolder[0])
 
@@ -252,6 +265,8 @@ define [
 					id:   file._id
 					type: "file"
 					selected: (file._id == @selected_entity_id)
+					linkedFileData: file.linkedFileData
+					created: file.created
 				}
 
 			for childFolder in rawFolder.folders or []
@@ -355,6 +370,34 @@ define [
 				_csrf: window.csrfToken
 			}
 
+		createLinkedFile: (name, parent_folder = @getCurrentFolder(), provider, data) ->
+			# check if a doc/file/folder already exists with this name
+			if @existsInThisFolder parent_folder, name
+				return @nameExistsError()
+			# We'll wait for the socket.io notification to actually
+			# add the file for us.
+			return @ide.$http.post "/project/#{@ide.project_id}/linked_file", {
+				name: name,
+				parent_folder_id: parent_folder?.id
+				provider,
+				data,
+				_csrf: window.csrfToken
+			}
+
+		refreshLinkedFile: (file) ->
+			parent_folder = @_findParentFolder(file)
+			data = file.linkedFileData
+			provider = data?.provider
+			return if !provider?
+			console.log 'refreshLinkedFile', {parent_folder, provider, data}
+			return @ide.$http.post "/project/#{@ide.project_id}/linked_file", {
+				name: file.name,
+				parent_folder_id: parent_folder?.id
+				provider,
+				data,
+				_csrf: window.csrfToken
+			}
+
 		renameEntity: (entity, name, callback = (error) ->) ->
 			return if entity.name == name
 			return if name.length >= 150

services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee

@@ -30,6 +30,19 @@ define [
 				}
 			)
 
+		$scope.openLinkedFileModal = window.openLinkedFileModal = () ->
+			unless 'url' in window.data.enabledLinkedFileTypes
+				console.warn("Url linked files are not enabled")
+				return
+			$modal.open(
+				templateUrl: "linkedFileModalTemplate"
+				controller:  "LinkedFileModalController"
+				scope: $scope
+				resolve: {
+					parent_folder: () -> ide.fileTreeManager.getCurrentFolder()
+				}
+			)
+
 		$scope.orderByFoldersFirst = (entity) ->
 			return '0' if entity?.type == "folder"
 			return '1'
@@ -186,4 +199,47 @@ define [
 
 			$scope.cancel = () ->
 				$modalInstance.dismiss('cancel')
-	]
+	]
+
+	App.controller "LinkedFileModalController", [
+		"$scope", "ide", "$modalInstance", "$timeout", "parent_folder",
+		($scope,   ide,   $modalInstance,   $timeout,   parent_folder) ->
+			$scope.inputs =
+				name: ""
+				url: ""
+			$scope.nameChangedByUser = false
+			$scope.state =
+				inflight: false
+
+			$modalInstance.opened.then () ->
+				$timeout () ->
+					$scope.$broadcast "open"
+				, 200
+
+			$scope.$watch "inputs.url", (url) ->
+				if url? and url != "" and !$scope.nameChangedByUser
+					url = url.replace("://", "") # Ignore http:// etc
+					parts = url.split("/").reverse()
+					if parts.length > 1 # Wait for at one /
+						$scope.inputs.name = parts[0]
+
+			$scope.create = () ->
+				{name, url} = $scope.inputs
+				if !name? or name.length == 0
+					return
+				if !url? or url.length == 0
+					return
+				$scope.state.inflight = true
+				ide.fileTreeManager
+					.createLinkedFile(name, parent_folder, 'url', {url})
+					.then () ->
+						$scope.state.inflight = false
+						$modalInstance.close()
+					.catch (response)->
+						{ data } = response
+						$scope.error = data
+						$scope.state.inflight = false
+
+			$scope.cancel = () ->
+				$modalInstance.dismiss('cancel')
+	]

services/web/public/stylesheets/app/editor/binary-file.less

@@ -22,14 +22,15 @@
 		font-size: 24px;
 		color: @gray;
 	}
-	.bib-loading {
+	.text-loading {
 		font-size: 24px;
 		color: @gray;
 		margin-bottom: 12px;
 	}
-	.bib-preview {
+	.text-preview {
 		margin-bottom: 12px;
 		.scroll-container {
+			background-color: white;
 			font-size: 0.8em;
 			line-height: 1.1em;
 			overflow: auto;
@@ -43,5 +44,8 @@
 			font-family: monospace;
 		}
 	}
+	.linked-file-icon {
+		color: @blue
+	}
 
 }

services/web/public/stylesheets/app/editor/file-tree.less

@@ -101,6 +101,18 @@
 			i.fa {
 				color: @file-tree-item-icon-color;
 				font-size: 14px;
+				&.linked-file-highlight {
+					&when (@is-overleaf = true) {
+						color: white;
+					}
+					&when (@is-overleaf = false) {
+						color: @blue;
+					}
+					position: relative;
+					top: 4px;
+					left: -8px;
+					font-size: 12px;
+				}
 			}
 
 			i.fa-folder-open, i.fa-folder {
@@ -129,6 +141,9 @@
 						.entity-menu-toggle i.fa {
 							color: #FFF;
 						}
+						> i.fa i.linked-file-highlight {
+							color: @blue;
+						}
 						color: #FFF;
 						font-weight: bold;
 						background-color: @file-tree-multiselect-bg;
@@ -191,6 +206,9 @@
 					.entity-menu-toggle i.fa {
 						color: #FFF;
 					}
+					> i.fa i.linked-file-highlight {
+						color: @blue;
+					}
 					background-color: @file-tree-item-selected-bg;
 					font-weight: bold;
 					padding-right: 32px;

services/web/test/acceptance/coffee/LinkedFilesTests.coffee

@@ -0,0 +1,184 @@
+async = require "async"
+expect = require("chai").expect
+_ = require 'underscore'
+
+MockFileStoreApi = require './helpers/MockFileStoreApi'
+request = require "./helpers/request"
+User = require "./helpers/User"
+
+
+express = require("express")
+LinkedUrlProxy = express()
+LinkedUrlProxy.get "/", (req, res, next) =>
+	if req.query.url == 'http://example.com/foo'
+		res.send('foo foo foo')
+	else if req.query.url == 'http://example.com/bar'
+		res.send('bar bar bar')
+	else
+		res.sendStatus(404)
+
+describe "LinkedFiles", ->
+	before (done) ->
+		LinkedUrlProxy.listen 6543, (error) =>
+			return done(error) if error?
+			@owner = new User() 
+			@owner.login done
+
+	describe "creating a URL based linked file", ->
+		before (done) ->
+			@owner.createProject "url-linked-files-project", {template: "blank"}, (error, project_id) =>
+				throw error if error?
+				@project_id = project_id
+				@owner.getProject project_id, (error, project) =>
+					throw error if error?
+					@project = project
+					@root_folder_id = project.rootFolder[0]._id.toString()
+					done()
+
+		it "should download the URL and create a file with the contents and linkedFileData", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: 'http://example.com/foo'
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-1'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 204
+				@owner.getProject @project_id, (error, project) =>
+					throw error if error?
+					file = project.rootFolder[0].fileRefs[0]
+					expect(file.linkedFileData).to.deep.equal({
+						provider: 'url'
+						url: 'http://example.com/foo'
+					})
+					@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
+						throw error if error?
+						expect(response.statusCode).to.equal 200
+						expect(body).to.equal "foo foo foo"
+						done()
+
+		it "should replace and update a URL based linked file", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: 'http://example.com/foo'
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-2'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 204
+				@owner.request.post {
+					url: "/project/#{@project_id}/linked_file",
+					json:
+						provider: 'url'
+						data: {
+							url: 'http://example.com/bar'
+						}
+						parent_folder_id: @root_folder_id
+						name: 'url-test-file-2'
+				}, (error, response, body) =>
+					throw error if error?
+					expect(response.statusCode).to.equal 204
+					@owner.getProject @project_id, (error, project) =>
+						throw error if error?
+						file = project.rootFolder[0].fileRefs[1]
+						expect(file.linkedFileData).to.deep.equal({
+							provider: 'url'
+							url: 'http://example.com/bar'
+						})
+						@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
+							throw error if error?
+							expect(response.statusCode).to.equal 200
+							expect(body).to.equal "bar bar bar"
+							done()
+
+		it "should return an error if the URL does not succeed", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: 'http://example.com/does-not-exist'
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-3'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 422 # unprocessable
+				expect(body).to.equal(
+					"Your URL could not be reached (404 status code). Please check it and try again."
+				)
+				done()
+
+		it "should return an error if the URL is invalid", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: "!^$%"
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-4'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 422 # unprocessable
+				expect(body).to.equal(
+					"Your URL is not valid. Please check it and try again."
+				)
+				done()
+
+		it "should return an error if the URL uses a non-http protocol", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: "ftp://localhost"
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-5'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 422 # unprocessable
+				expect(body).to.equal(
+					"Your URL is not valid. Please check it and try again."
+				)
+				done()
+
+		it "should accept a URL withuot a leading http://, and add it", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: 'example.com/foo'
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-6'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 204
+				@owner.getProject @project_id, (error, project) =>
+					throw error if error?
+					file = _.find project.rootFolder[0].fileRefs, (file) ->
+						file.name == 'url-test-file-6'
+					expect(file.linkedFileData).to.deep.equal({
+						provider: 'url'
+						url: 'http://example.com/foo'
+					})
+					@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
+						throw error if error?
+						expect(response.statusCode).to.equal 200
+						expect(body).to.equal "foo foo foo"
+						done()
+
+		# TODO: Add test for asking for host that return ENOTFOUND
+		# (This will probably end up handled by the proxy)

services/web/test/acceptance/coffee/helpers/MockFileStoreApi.coffee

@@ -6,14 +6,22 @@ module.exports = MockFileStoreApi =
 
 	run: () ->
 		app.post "/project/:project_id/file/:file_id", (req, res, next) =>
-			req.on 'data', ->
+			chunks = []
+			req.on 'data', (chunk) ->
+				chunks.push(chunk)
 
 			req.on 'end', =>
+				content = Buffer.concat(chunks).toString()
 				{project_id, file_id} = req.params
 				@files[project_id] ?= {}
-				@files[project_id][file_id] = { content : "test-file-content" }
+				@files[project_id][file_id] = { content }
 				res.sendStatus 200
 
+		app.get "/project/:project_id/file/:file_id", (req, res, next) =>
+			{project_id, file_id} = req.params
+			{ content } = @files[project_id][file_id]
+			res.send content
+
 		app.listen 3009, (error) ->
 			throw error if error?
 		.on "error", (error) ->

services/web/test/unit/coffee/Editor/EditorControllerTests.coffee

@@ -18,6 +18,7 @@ describe "EditorController", ->
 		@file = _id: @file_id ="dasdkjk"
 		@fileName = "file.png"
 		@fsPath = "/folder/file.png"
+		@linkedFileData = {provider: 'url'}
 
 		@folder_id = "123ksajdn"
 		@folder = _id: @folder_id
@@ -66,16 +67,16 @@ describe "EditorController", ->
 	describe 'addFile', ->
 		beforeEach ->
 			@ProjectEntityUpdateHandler.addFile = sinon.stub().yields(null, @file, @folder_id)
-			@EditorController.addFile @project_id, @folder_id, @fileName, @fsPath, @source, @user_id, @callback
+			@EditorController.addFile @project_id, @folder_id, @fileName, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 		it 'should add the folder using the project entity handler', ->
 			@ProjectEntityUpdateHandler.addFile
-				.calledWith(@project_id, @folder_id, @fileName, @fsPath, @user_id)
+				.calledWith(@project_id, @folder_id, @fileName, @fsPath, @linkedFileData, @user_id)
 				.should.equal true
 
 		it 'should send the update of a new folder out to the users in the project', ->
 			@EditorRealTimeController.emitToRoom
-				.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source)
+				.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source, @linkedFileData)
 				.should.equal true
 
 		it 'calls the callback', ->
@@ -107,11 +108,11 @@ describe "EditorController", ->
 	describe 'upsertFile', ->
 		beforeEach ->
 			@ProjectEntityUpdateHandler.upsertFile = sinon.stub().yields(null, @file, false)
-			@EditorController.upsertFile @project_id, @folder_id, @fileName, @fsPath, @source, @user_id, @callback
+			@EditorController.upsertFile @project_id, @folder_id, @fileName, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 		it 'upserts the file using the project entity handler', ->
 			@ProjectEntityUpdateHandler.upsertFile
-				.calledWith(@project_id, @folder_id, @fileName, @fsPath, @user_id)
+				.calledWith(@project_id, @folder_id, @fileName, @fsPath, @linkedFileData, @user_id)
 				.should.equal true
 
 		it 'returns the file', ->
@@ -120,11 +121,11 @@ describe "EditorController", ->
 		describe 'file does not exist', ->
 			beforeEach ->
 				@ProjectEntityUpdateHandler.upsertFile = sinon.stub().yields(null, @file, true)
-				@EditorController.upsertFile @project_id, @folder_id, @fileName, @fsPath, @source, @user_id, @callback
+				@EditorController.upsertFile @project_id, @folder_id, @fileName, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 			it 'should send the update out to users in the project', ->
 				@EditorRealTimeController.emitToRoom
-					.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source)
+					.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source, @linkedFileData)
 					.should.equal true
 
 	describe "upsertDocWithPath", ->
@@ -171,21 +172,21 @@ describe "EditorController", ->
 			@filePath = '/folder/file'
 
 			@ProjectEntityUpdateHandler.upsertFileWithPath = sinon.stub().yields(null, @file, false, [], @folder)
-			@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @source, @user_id, @callback
+			@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 		it 'upserts the file using the project entity handler', ->
 			@ProjectEntityUpdateHandler.upsertFileWithPath
-				.calledWith(@project_id, @filePath, @fsPath)
+				.calledWith(@project_id, @filePath, @fsPath, @linkedFileData)
 				.should.equal true
 
 		describe 'file does not exist', ->
 			beforeEach ->
 				@ProjectEntityUpdateHandler.upsertFileWithPath = sinon.stub().yields(null, @file, true, [], @folder)
-				@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @source, @user_id, @callback
+				@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 			it 'should send the update for the file out to users in the project', ->
 				@EditorRealTimeController.emitToRoom
-					.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source)
+					.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source, @linkedFileData)
 					.should.equal true
 
 		describe 'folders required for file do not exist', ->
@@ -195,7 +196,7 @@ describe "EditorController", ->
 					@folderB = { _id: 3, parentFolder_id: 2}
 				]
 				@ProjectEntityUpdateHandler.upsertFileWithPath = sinon.stub().yields(null, @file, true, folders, @folderB)
-				@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @source, @user_id, @callback
+				@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 			it 'should send the update for each folder to users in the project', ->
 				@EditorRealTimeController.emitToRoom

services/web/test/unit/coffee/Project/ProjectCreationHandlerTests.coffee

@@ -34,7 +34,7 @@ describe 'ProjectCreationHandler', ->
 				{@name} = options
 		@ProjectEntityUpdateHandler =
 			addDoc: sinon.stub().callsArgWith(5, null, {_id: docId})
-			addFile: sinon.stub().callsArg(5)
+			addFile: sinon.stub().callsArg(6)
 			setRootDoc: sinon.stub().callsArg(2)
 		@ProjectDetailsHandler =
 			validateProjectName: sinon.stub().yields()
@@ -208,6 +208,7 @@ describe 'ProjectCreationHandler', ->
 				.calledWith(
 					project_id, rootFolderId, "universe.jpg",
 					Path.resolve(__dirname + "/../../../../app/templates/project_files/universe.jpg"),
+					null,
 					ownerId
 				)
 				.should.equal true

services/web/test/unit/coffee/Project/ProjectEditorHandlerTests.coffee

@@ -33,7 +33,7 @@ describe "ProjectEditorHandler", ->
 					fileRefs : [{
 						_id     : "file-id"
 						name    : "image.png"
-						created : new Date()
+						created : @created = new Date()
 						size    : 1234
 					}]
 					folders  : []
@@ -141,7 +141,7 @@ describe "ProjectEditorHandler", ->
 			it "should include files in the project", ->
 				@result.rootFolder[0].folders[0].fileRefs[0]._id.should.equal "file-id"
 				@result.rootFolder[0].folders[0].fileRefs[0].name.should.equal "image.png"
-				should.not.exist @result.rootFolder[0].folders[0].fileRefs[0].created
+				@result.rootFolder[0].folders[0].fileRefs[0].created.should.equal @created
 				should.not.exist @result.rootFolder[0].folders[0].fileRefs[0].size
 
 			it "should include docs in the project but not the lines", ->

services/web/test/unit/coffee/Project/ProjectEntityMongoUpdateHandlerTests.coffee

@@ -97,10 +97,11 @@ describe 'ProjectEntityMongoUpdateHandler', ->
 		beforeEach ->
 			@file = _id: file_id
 			@path = mongo: 'file.png'
+			@linkedFileData = {provider: 'url'}
 			@ProjectLocator.findElement = sinon.stub().yields(null, @file, @path)
 			@ProjectModel.update = sinon.stub().yields()
 
-			@subject.replaceFile project_id, file_id, @callback
+			@subject.replaceFile project_id, file_id, @linkedFileData, @callback
 
 		it 'gets the project', ->
 			@ProjectGetter.getProjectWithoutLock
@@ -118,7 +119,7 @@ describe 'ProjectEntityMongoUpdateHandler', ->
 					{ _id: project_id },
 					{
 						'$inc': { 'file.png.rev': 1, 'version': 1 }
-						'$set': { 'file.png.created': new Date() }
+						'$set': { 'file.png.created': new Date(), 'file.png.linkedFileData': @linkedFileData }
 					}
 					{}
 				)

services/web/test/unit/coffee/Project/ProjectEntityUpdateHandlerTests.coffee

@@ -40,6 +40,8 @@ describe 'ProjectEntityUpdateHandler', ->
 		@fileName = "something.jpg"
 		@fileSystemPath = "somehintg"
 
+		@linkedFileData = {provider: 'url'}
+
 		@source = 'editor'
 		@callback = sinon.stub()
 		@ProjectEntityUpdateHandler = SandboxedModule.require modulePath, requires:
@@ -296,11 +298,11 @@ describe 'ProjectEntityUpdateHandler', ->
 			@newFile = _id: file_id
 			@ProjectEntityUpdateHandler.addFileWithoutUpdatingHistory =
 				withoutLock: sinon.stub().yields(null, @newFile, folder_id, @path, @fileUrl)
-			@ProjectEntityUpdateHandler.addFile project_id, folder_id, @docName, @fileSystemPath, userId, @callback
+			@ProjectEntityUpdateHandler.addFile project_id, folder_id, @docName, @fileSystemPath, @linkedFileData, userId, @callback
 
 		it "creates the doc without history", () ->
 			@ProjectEntityUpdateHandler.addFileWithoutUpdatingHistory.withoutLock
-				.calledWith(project_id, folder_id, @docName, @fileSystemPath, userId)
+				.calledWith(project_id, folder_id, @docName, @fileSystemPath, @linkedFileData, userId)
 				.should.equal true
 
 		it "sends the change in project structure to the doc updater", () ->
@@ -320,7 +322,7 @@ describe 'ProjectEntityUpdateHandler', ->
 			@project = _id: project_id, name: 'some project'
 			@ProjectEntityMongoUpdateHandler.replaceFile = sinon.stub().yields(null, @newFile, @project, fileSystem: @path)
 
-			@ProjectEntityUpdateHandler.replaceFile project_id, file_id, @fileSystemPath, userId, @callback
+			@ProjectEntityUpdateHandler.replaceFile project_id, file_id, @fileSystemPath, @linkedFileData, userId, @callback
 
 		it 'uploads a new version of the file', ->
 			@FileStoreHandler.uploadFileFromDisk
@@ -329,7 +331,7 @@ describe 'ProjectEntityUpdateHandler', ->
 
 		it 'replaces the file in mongo', ->
 			@ProjectEntityMongoUpdateHandler.replaceFile
-				.calledWith(project_id, file_id)
+				.calledWith(project_id, file_id, @linkedFileData)
 				.should.equal true
 
 		it 'notifies the tpds', ->
@@ -497,7 +499,7 @@ describe 'ProjectEntityUpdateHandler', ->
 		describe 'upserting into an invalid folder', ->
 			beforeEach ->
 				@ProjectLocator.findElement = sinon.stub().yields()
-				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, userId, @callback
+				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, @linkedFileData, userId, @callback
 
 			it 'returns an error', ->
 				errorMatcher = sinon.match.instanceOf(Error)
@@ -511,11 +513,11 @@ describe 'ProjectEntityUpdateHandler', ->
 				@ProjectLocator.findElement = sinon.stub().yields(null, @folder)
 				@ProjectEntityUpdateHandler.replaceFile = withoutLock: sinon.stub().yields(null, @newFile)
 
-				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, userId, @callback
+				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, @linkedFileData, userId, @callback
 
 			it 'replaces the file', ->
 				@ProjectEntityUpdateHandler.replaceFile.withoutLock
-					.calledWith(project_id, file_id, @fileSystemPath, userId)
+					.calledWith(project_id, file_id, @fileSystemPath, @linkedFileData, userId)
 					.should.equal true
 
 			it 'returns the file', ->
@@ -528,7 +530,7 @@ describe 'ProjectEntityUpdateHandler', ->
 				@ProjectLocator.findElement = sinon.stub().yields(null, @folder)
 				@ProjectEntityUpdateHandler.addFile = withoutLock: sinon.stub().yields(null, @newFile)
 
-				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, userId, @callback
+				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, @linkedFileData, userId, @callback
 
 			it 'tries to find the folder', ->
 				@ProjectLocator.findElement
@@ -537,7 +539,7 @@ describe 'ProjectEntityUpdateHandler', ->
 
 			it 'adds the file', ->
 				@ProjectEntityUpdateHandler.addFile.withoutLock
-					.calledWith(project_id, folder_id, @fileName, @fileSystemPath, userId)
+					.calledWith(project_id, folder_id, @fileName, @fileSystemPath, @linkedFileData, userId)
 					.should.equal true
 
 			it 'returns the file', ->
@@ -584,7 +586,7 @@ describe 'ProjectEntityUpdateHandler', ->
 			@ProjectEntityUpdateHandler.upsertFile =
 				withoutLock: sinon.stub().yields(null, @file, @isNewFile)
 
-			@ProjectEntityUpdateHandler.upsertFileWithPath project_id, @path, @fileSystemPath, userId, @callback
+			@ProjectEntityUpdateHandler.upsertFileWithPath project_id, @path, @fileSystemPath, @linkedFileData, userId, @callback
 
 		it 'creates any necessary folders', ->
 			@ProjectEntityUpdateHandler.mkdirp.withoutLock
@@ -593,7 +595,7 @@ describe 'ProjectEntityUpdateHandler', ->
 
 		it 'upserts the file', ->
 			@ProjectEntityUpdateHandler.upsertFile.withoutLock
-				.calledWith(project_id, @folder._id, 'file.png', @fileSystemPath, userId)
+				.calledWith(project_id, @folder._id, 'file.png', @fileSystemPath, @linkedFileData, userId)
 				.should.equal true
 
 		it 'calls the callback', ->

services/web/test/unit/coffee/Subscription/SubscriptionControllerTests.coffee

@@ -74,6 +74,7 @@ describe "SubscriptionController", ->
 			"settings-sharelatex": @settings
 			"./SubscriptionDomainHandler":@SubscriptionDomainHandler
 			"../User/UserGetter": @UserGetter
+			"./RecurlyWrapper": @RecurlyWrapper = {}
 
 
 		@res = new MockResponse()
@@ -117,6 +118,7 @@ describe "SubscriptionController", ->
 	describe "paymentPage", ->
 		beforeEach ->
 			@req.headers = {}
+			@RecurlyWrapper.sign = sinon.stub().yields(null, @signature = "signature")
 			@SubscriptionHandler.validateNoSubscriptionInRecurly = sinon.stub().yields(null, true)
 			@GeoIpLookup.getCurrencyCode.callsArgWith(1, null, @stubbedCurrencyCode)
 

services/web/test/unit/coffee/ThirdPartyDataStore/UpdateMergerTests.coffee

@@ -15,18 +15,20 @@ describe 'UpdateMerger :', ->
 				err: ->
 			'../Editor/EditorController': @EditorController = {}
 			'../Uploads/FileTypeManager':@FileTypeManager = {}
+			'../../infrastructure/FileWriter': @FileWriter = {}
 			'settings-sharelatex':{path:{dumpPath:"dump_here"}}
 		@project_id = "project_id_here"
 		@user_id = "mock-user-id"
 
 		@docPath = "/folder/doc.tex"
 		@filePath = "/folder/file.png"
+		@linkedFileData = {provider: 'url'}
 
 		@fsPath = "/tmp/file/path"
 
 		@source = "dropbox"
 		@updateRequest = new BufferedStream()
-		@updateMerger.p.writeStreamToDisk = sinon.stub().yields(null, @fsPath)
+		@FileWriter.writeStreamToDisk = sinon.stub().yields(null, @fsPath)
 		@callback = sinon.stub()
 
 	describe 'mergeUpdate', ->
@@ -94,5 +96,5 @@ describe 'UpdateMerger :', ->
 
 			it 'should upsert the file in the editor controller', ->
 				@EditorController.upsertFileWithPath
-					.calledWith(@project_id, @filePath, @fsPath, @source, @user_id)
+					.calledWith(@project_id, @filePath, @fsPath, null, @source, @user_id)
 					.should.equal true

services/web/test/unit/coffee/Uploads/FileSystemImportManagerTests.coffee

@@ -78,12 +78,12 @@ describe "FileSystemImportManager", ->
 
 	describe "addFile with replace set to false", ->
 		beforeEach ->
-			@EditorController.addFile = sinon.stub().callsArg(6)
+			@EditorController.addFile = sinon.stub().yields()
 			@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
 			@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, false, @callback
 
 		it "should add the file", ->
-			@EditorController.addFile.calledWith(@project_id, @folder_id, @name, @path_on_disk, "upload", @user_id)
+			@EditorController.addFile.calledWith(@project_id, @folder_id, @name, @path_on_disk, null, "upload", @user_id)
 				.should.equal true
 
 	describe "addFile with symlink", ->
@@ -105,7 +105,7 @@ describe "FileSystemImportManager", ->
 
 		it "should add the file", ->
 			@EditorController.upsertFile
-				.calledWith(@project_id, @folder_id, @name, @path_on_disk, "upload", @user_id)
+				.calledWith(@project_id, @folder_id, @name, @path_on_disk, null, "upload", @user_id)
 				.should.equal true
 
 	describe "addFolder", ->