First pass at URL based linked files

services/web/app/coffee/Features/Editor/EditorController.coffee

@@ -23,11 +23,11 @@ module.exports = EditorController =
 			EditorRealTimeController.emitToRoom(project_id, 'reciveNewDoc', folder_id, doc, source)
 			callback(err, doc)
 
-	addFile: (project_id, folder_id, fileName, fsPath, source, user_id, callback = (error, file)->)->
+	addFile: (project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id, callback = (error, file)->)->
 		fileName = fileName.trim()
-		logger.log {project_id, folder_id, fileName, fsPath}, "sending new file to project"
+		logger.log {project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id}, "sending new file to project"
 		Metrics.inc "editor.add-file"
-		ProjectEntityUpdateHandler.addFile project_id, folder_id, fileName, fsPath, user_id, (err, fileRef, folder_id)=>
+		ProjectEntityUpdateHandler.addFile project_id, folder_id, fileName, fsPath, linkedFileData, user_id, (err, fileRef, folder_id)=>
 			if err?
 				logger.err err:err, project_id:project_id, folder_id:folder_id, fileName:fileName, "error adding file without lock"
 				return callback(err)
@@ -40,8 +40,8 @@ module.exports = EditorController =
 				EditorRealTimeController.emitToRoom(project_id, 'reciveNewDoc', folder_id, doc, source)
 			callback err, doc
 
-	upsertFile: (project_id, folder_id, fileName, fsPath, source, user_id, callback = (err, file) ->) ->
-		ProjectEntityUpdateHandler.upsertFile project_id, folder_id, fileName, fsPath, user_id, (err, file, didAddFile) ->
+	upsertFile: (project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id, callback = (err, file) ->) ->
+		ProjectEntityUpdateHandler.upsertFile project_id, folder_id, fileName, fsPath, linkedFileData, user_id, (err, file, didAddFile) ->
 			return callback(err) if err?
 			if didAddFile
 				EditorRealTimeController.emitToRoom project_id, 'reciveNewFile', folder_id, file, source
@@ -56,8 +56,8 @@ module.exports = EditorController =
 					EditorRealTimeController.emitToRoom project_id, 'reciveNewDoc', lastFolder._id, doc, source
 				callback()
 
-	upsertFileWithPath: (project_id, elementPath, fsPath, source, user_id, callback) ->
-		ProjectEntityUpdateHandler.upsertFileWithPath project_id, elementPath, fsPath, user_id, (err, file, didAddFile, newFolders, lastFolder) ->
+	upsertFileWithPath: (project_id, elementPath, fsPath, linkedFileData, source, user_id, callback) ->
+		ProjectEntityUpdateHandler.upsertFileWithPath project_id, elementPath, fsPath, linkedFileData, user_id, (err, file, didAddFile, newFolders, lastFolder) ->
 			return callback(err) if err?
 			EditorController._notifyProjectUsersOfNewFolders project_id, newFolders, (err) ->
 				return callback(err) if err?

services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee

@@ -0,0 +1,25 @@
+AuthenticationController = require '../Authentication/AuthenticationController'
+EditorController = require '../Editor/EditorController'
+
+module.exports = LinkedFilesController = {
+	Agents: {
+		url: require('./UrlAgent')
+	}
+
+	createLinkedFile: (req, res, next) ->
+		{project_id} = req.params
+		{name, provider, data, parent_folder_id} = req.body
+		user_id = AuthenticationController.getLoggedInUserId(req)
+
+		if !LinkedFilesController.Agents.hasOwnProperty(provider)
+			return res.send(400)
+		Agent = LinkedFilesController.Agents[provider]
+
+		linkedFileData = Agent.sanitizeData(data)
+		linkedFileData.provider = provider
+		Agent.writeIncomingFileToDisk project_id, linkedFileData, user_id, (error, fsPath) ->
+			return next(error) if error?
+			EditorController.upsertFile project_id, parent_folder_id, name, fsPath, linkedFileData, "upload", user_id, (error) ->
+				return next(error) if error?
+				res.send(204) # created
+}

services/web/app/coffee/Features/LinkedFiles/LinkedFilesRouter.coffee

@@ -0,0 +1,11 @@
+AuthorizationMiddlewear = require('../Authorization/AuthorizationMiddlewear')
+AuthenticationController = require('../Authentication/AuthenticationController')
+LinkedFilesController = require "./LinkedFilesController"
+
+module.exports =
+	apply: (webRouter) ->
+		webRouter.post '/project/:project_id/linked_file',
+			AuthenticationController.requireLogin(),
+			AuthorizationMiddlewear.ensureUserCanWriteProjectContent,
+			LinkedFilesController.createLinkedFile
+

services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee

@@ -0,0 +1,15 @@
+request = require 'request'
+FileWriter = require('../../infrastructure/FileWriter')
+
+module.exports = UrlAgent = {
+	sanitizeData: (data) ->
+		return {
+			url: data.url
+		}
+
+	writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
+		# TODO: proxy through external API
+		url = data.url
+		readStream = request.get(url)
+		FileWriter.writeStreamToDisk project_id, readStream, callback
+}

services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee

@@ -79,7 +79,7 @@ module.exports = ProjectCreationHandler =
 							callback(error)
 				(callback) ->
 					universePath = Path.resolve(__dirname + "/../../../templates/project_files/universe.jpg")
-					ProjectEntityUpdateHandler.addFile project._id, project.rootFolder[0]._id, "universe.jpg", universePath, owner_id, callback
+					ProjectEntityUpdateHandler.addFile project._id, project.rootFolder[0]._id, "universe.jpg", universePath, null, owner_id, callback
 			], (error) ->
 				callback(error, project)
 

services/web/app/coffee/Features/Project/ProjectEntityMongoUpdateHandler.coffee

@@ -48,7 +48,7 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
 			self._confirmFolder project, folder_id, (folder_id)->
 				self._putElement project, folder_id, fileRef, "file", callback
 
-	replaceFile: wrapWithLock (project_id, file_id, callback) ->
+	replaceFile: wrapWithLock (project_id, file_id, linkedFileData, callback) ->
 		ProjectGetter.getProjectWithoutLock project_id, {rootFolder: true, name:true}, (err, project) ->
 			return callback(err) if err?
 			ProjectLocator.findElement {project:project, element_id: file_id, type: 'file'}, (err, fileRef, path)=>
@@ -63,6 +63,7 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
 				inc['version'] = 1
 				set = {}
 				set["#{path.mongo}.created"] = new Date()
+				set["#{path.mongo}.linkedFileData"] = linkedFileData
 				update =
 					"$inc": inc
 					"$set": set

services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee

@@ -129,8 +129,8 @@ module.exports = ProjectEntityUpdateHandler = self =
 				return callback(error) if error?
 				callback null, doc, folder_id
 
-	addFile: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (error, fileRef, folder_id) ->)->
-		self.addFileWithoutUpdatingHistory.withoutLock project_id, folder_id, fileName, fsPath, userId, (error, fileRef, folder_id, path, fileStoreUrl) ->
+	addFile: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (error, fileRef, folder_id) ->)->
+		self.addFileWithoutUpdatingHistory.withoutLock project_id, folder_id, fileName, fsPath, linkedFileData, userId, (error, fileRef, folder_id, path, fileStoreUrl) ->
 			return callback(error) if error?
 			newFiles = [
 				file: fileRef
@@ -141,10 +141,10 @@ module.exports = ProjectEntityUpdateHandler = self =
 				return callback(error) if error?
 				callback null, fileRef, folder_id
 
-	replaceFile: wrapWithLock (project_id, file_id, fsPath, userId, callback)->
+	replaceFile: wrapWithLock (project_id, file_id, fsPath, linkedFileData, userId, callback)->
 		FileStoreHandler.uploadFileFromDisk project_id, file_id, fsPath, (err, fileStoreUrl)->
 			return callback(err) if err?
-			ProjectEntityMongoUpdateHandler.replaceFile project_id, file_id, (err, fileRef, project, path) ->
+			ProjectEntityMongoUpdateHandler.replaceFile project_id, file_id, linkedFileData, (err, fileRef, project, path) ->
 				return callback(err) if err?
 				newFiles = [
 					file: fileRef
@@ -180,7 +180,7 @@ module.exports = ProjectEntityUpdateHandler = self =
 					return callback(err) if err?
 					callback(null, doc, folder_id, result?.path?.fileSystem)
 
-	addFileWithoutUpdatingHistory: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (error, fileRef, folder_id, path, fileStoreUrl) ->)->
+	addFileWithoutUpdatingHistory: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (error, fileRef, folder_id, path, fileStoreUrl) ->)->
 		# This method should never be called directly, except when importing a project
 		# from Overleaf. It skips sending updates to the project history, which will break
 		# the history unless you are making sure it is updated in some other way.
@@ -188,7 +188,10 @@ module.exports = ProjectEntityUpdateHandler = self =
 		if not SafePath.isCleanFilename fileName
 			return callback new Errors.InvalidNameError("invalid element name")
 
-		fileRef = new File name : fileName
+		fileRef = new File(
+			name: fileName
+			linkedFileData: linkedFileData
+		)
 		FileStoreHandler.uploadFileFromDisk project_id, fileRef._id, fsPath, (err, fileStoreUrl)->
 			if err?
 				logger.err err:err, project_id: project_id, folder_id: folder_id, file_name: fileName, fileRef:fileRef, "error uploading image to s3"
@@ -221,7 +224,7 @@ module.exports = ProjectEntityUpdateHandler = self =
 					return callback(err) if err?
 					callback null, doc, !existingDoc?
 
-	upsertFile: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (err, file, isNewFile)->)->
+	upsertFile: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (err, file, isNewFile)->)->
 		ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
 			return callback(error) if error?
 			return callback(new Error("Couldn't find folder")) if !folder?
@@ -231,11 +234,11 @@ module.exports = ProjectEntityUpdateHandler = self =
 					existingFile = fileRef
 					break
 			if existingFile?
-				self.replaceFile.withoutLock project_id, existingFile._id, fsPath, userId, (err) ->
+				self.replaceFile.withoutLock project_id, existingFile._id, fsPath, linkedFileData, userId, (err) ->
 					return callback(err) if err?
 					callback null, existingFile, !existingFile?
 			else
-				self.addFile.withoutLock project_id, folder_id, fileName, fsPath, userId, (err, file) ->
+				self.addFile.withoutLock project_id, folder_id, fileName, fsPath, linkedFileData, userId, (err, file) ->
 					return callback(err) if err?
 					callback null, file, !existingFile?
 
@@ -248,12 +251,12 @@ module.exports = ProjectEntityUpdateHandler = self =
 				return callback(err) if err?
 				callback null, doc, isNewDoc, newFolders, folder
 
-	upsertFileWithPath: wrapWithLock (project_id, elementPath, fsPath, userId, callback) ->
+	upsertFileWithPath: wrapWithLock (project_id, elementPath, fsPath, linkedFileData, userId, callback) ->
 		fileName = path.basename(elementPath)
 		folderPath = path.dirname(elementPath)
 		self.mkdirp.withoutLock project_id, folderPath, (err, newFolders, folder) ->
 			return callback(err) if err?
-			self.upsertFile.withoutLock project_id, folder._id, fileName, fsPath, userId, (err, file, isNewFile) ->
+			self.upsertFile.withoutLock project_id, folder._id, fileName, fsPath, linkedFileData, userId, (err, file, isNewFile) ->
 				return callback(err) if err?
 				callback null, file, isNewFile, newFolders, folder
 

services/web/app/coffee/Features/ThirdPartyDataStore/UpdateMerger.coffee

@@ -1,15 +1,14 @@
 _ = require('underscore')
 fs = require('fs')
 logger = require('logger-sharelatex')
-uuid = require('uuid')
 EditorController = require('../Editor/EditorController')
 FileTypeManager = require('../Uploads/FileTypeManager')
-Settings = require('settings-sharelatex')
+FileWriter = require('../../infrastructure/FileWriter')
 
 module.exports = UpdateMerger =
 	mergeUpdate: (user_id, project_id, path, updateRequest, source, callback = (error) ->)->
 		logger.log project_id:project_id, path:path, "merging update from tpds"
-		UpdateMerger.p.writeStreamToDisk project_id, updateRequest, (err, fsPath)->
+		FileWriter.writeStreamToDisk project_id, updateRequest, (err, fsPath)->
 			return callback(err) if err?
 			UpdateMerger._mergeUpdate user_id, project_id, path, fsPath, source, (mergeErr) ->
 				fs.unlink fsPath, (deleteErr) ->
@@ -44,29 +43,10 @@ module.exports = UpdateMerger =
 
 		processFile: (project_id, fsPath, path, source, user_id, callback)->
 			logger.log project_id:project_id, "processing file update from tpds"
-			EditorController.upsertFileWithPath project_id, path, fsPath, source, user_id, (err) ->
+			EditorController.upsertFileWithPath project_id, path, fsPath, null, source, user_id, (err) ->
 				logger.log project_id:project_id, "completed processing file update from tpds"
 				callback(err)
 
-		writeStreamToDisk: (project_id, stream, callback = (err, fsPath)->)->
-			dumpPath = "#{Settings.path.dumpFolder}/#{project_id}_#{uuid.v4()}"
-
-			writeStream = fs.createWriteStream(dumpPath)
-			stream.pipe(writeStream)
-
-			stream.on 'error', (err)->
-				logger.err {err, project_id, dumpPath},
-					"something went wrong with incoming tpds update stream"
-			writeStream.on 'error', (err)->
-				logger.err {err, project_id, dumpPath},
-					"something went wrong with writing tpds update to disk"
-
-			stream.on 'end', ->
-				logger.log {project_id, dumpPath}, "incoming tpds update stream ended"
-			writeStream.on "finish", ->
-				logger.log {project_id, dumpPath}, "tpds update write stream finished"
-				callback null, dumpPath
-
 		readFileIntoTextArray: (path, callback)->
 			fs.readFile path, "utf8", (error, content = "") ->
 				if error?

services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee

@@ -27,9 +27,9 @@ module.exports = FileSystemImportManager =
 				return callback("path is symlink")
 
 			if replace
-				EditorController.upsertFile project_id, folder_id, name, path, "upload", user_id, callback
+				EditorController.upsertFile project_id, folder_id, name, path, null, "upload", user_id, callback
 			else
-				EditorController.addFile project_id, folder_id, name, path, "upload", user_id, callback
+				EditorController.addFile project_id, folder_id, name, path, null, "upload", user_id, callback
 
 	addFolder: (user_id, project_id, folder_id, name, path, replace, callback = (error)-> ) ->
 		FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->

services/web/app/coffee/infrastructure/FileWriter.coffee

@@ -0,0 +1,23 @@
+fs = require 'fs'
+logger = require 'logger-sharelatex'
+uuid = require 'uuid'
+_ = require 'underscore'
+Settings = require 'settings-sharelatex'
+
+module.exports = 
+	writeStreamToDisk: (identifier, stream, callback = (error, fsPath) ->) ->
+		callback = _.once(callback)
+		fsPath = "#{Settings.path.dumpFolder}/#{identifier}_#{uuid.v4()}"
+
+		writeStream = fs.createWriteStream(fsPath)
+		stream.pipe(writeStream)
+
+		stream.on 'error', (err)->
+			logger.err {err, identifier, fsPath},	"[writeStreamToDisk] something went wrong with incoming stream"
+			callback(err)
+		writeStream.on 'error', (err)->
+			logger.err {err, identifier, fsPath},	"[writeStreamToDisk] something went wrong with writing to disk"
+			callback(err)
+		writeStream.on "finish", ->
+			logger.log {identifier, fsPath}, "[writeStreamToDisk] write stream finished"
+			callback null, fsPath

services/web/app/coffee/models/File.coffee

@@ -8,6 +8,7 @@ FileSchema = new Schema
 	name            :     type:String, default:''
 	created     	:     type:Date, default: () -> new Date()
 	rev 			:	  {type:Number, default:0}
+	linkedFileData: { type: Schema.Types.Mixed }
 
 mongoose.model 'File', FileSchema
 exports.File = mongoose.model 'File'

services/web/app/coffee/router.coffee

@@ -46,6 +46,7 @@ AnnouncementsController = require("./Features/Announcements/AnnouncementsControl
 MetaController = require('./Features/Metadata/MetaController')
 TokenAccessController = require('./Features/TokenAccess/TokenAccessController')
 Features = require('./infrastructure/Features')
+LinkedFilesRouter = require './Features/LinkedFiles/LinkedFilesRouter'
 
 logger = require("logger-sharelatex")
 _ = require("underscore")
@@ -77,6 +78,7 @@ module.exports = class Router
 		RealTimeProxyRouter.apply(webRouter, privateApiRouter)
 		ContactRouter.apply(webRouter, privateApiRouter)
 		AnalyticsRouter.apply(webRouter, privateApiRouter, publicApiRouter)
+		LinkedFilesRouter.apply(webRouter, privateApiRouter, publicApiRouter)
 
 		Modules.applyRouter(webRouter, privateApiRouter, publicApiRouter)
 

services/web/test/acceptance/coffee/LinkedFilesTests.coffee

@@ -0,0 +1,94 @@
+async = require "async"
+expect = require("chai").expect
+
+MockFileStoreApi = require './helpers/MockFileStoreApi'
+MockURLSource	= require './helpers/MockURLSource'
+request = require "./helpers/request"
+User = require "./helpers/User"
+
+MockURLSource.app.get "/foo", (req, res, next) =>
+	res.send('foo foo foo')
+MockURLSource.app.get "/bar", (req, res, next) =>
+	res.send('bar bar bar')
+
+describe "LinkedFiles", ->
+	before (done) ->
+		MockURLSource.run (error) =>
+			return done(error) if error?
+			@owner = new User() 
+			@owner.login done
+
+	describe "creating a URL based linked file", ->
+		before (done) ->
+			@owner.createProject "url-linked-files-project", {template: "blank"}, (error, project_id) =>
+				throw error if error?
+				@project_id = project_id
+				@owner.getProject project_id, (error, project) =>
+					throw error if error?
+					@project = project
+					@root_folder_id = project.rootFolder[0]._id.toString()
+					done()
+
+		it "should download the URL and create a file with the contents and linkedFileData", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: "http://localhost:6543/foo"
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-1'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 204
+				@owner.getProject @project_id, (error, project) =>
+					throw error if error?
+					file = project.rootFolder[0].fileRefs[0]
+					expect(file.linkedFileData).to.deep.equal({
+						provider: 'url'
+						url: "http://localhost:6543/foo"
+					})
+					@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
+						throw error if error?
+						expect(response.statusCode).to.equal 200
+						expect(body).to.equal "foo foo foo"
+						done()
+
+		it "should replace and update a URL based linked file", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: "http://localhost:6543/foo"
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-2'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 204
+				@owner.request.post {
+					url: "/project/#{@project_id}/linked_file",
+					json:
+						provider: 'url'
+						data: {
+							url: "http://localhost:6543/bar"
+						}
+						parent_folder_id: @root_folder_id
+						name: 'url-test-file-2'
+				}, (error, response, body) =>
+					throw error if error?
+					expect(response.statusCode).to.equal 204
+					@owner.getProject @project_id, (error, project) =>
+						throw error if error?
+						file = project.rootFolder[0].fileRefs[1]
+						expect(file.linkedFileData).to.deep.equal({
+							provider: 'url'
+							url: "http://localhost:6543/bar"
+						})
+						@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
+							throw error if error?
+							expect(response.statusCode).to.equal 200
+							expect(body).to.equal "bar bar bar"
+							done()

services/web/test/acceptance/coffee/helpers/MockFileStoreApi.coffee

@@ -6,14 +6,22 @@ module.exports = MockFileStoreApi =
 
 	run: () ->
 		app.post "/project/:project_id/file/:file_id", (req, res, next) =>
-			req.on 'data', ->
+			chunks = []
+			req.on 'data', (chunk) ->
+				chunks.push(chunk)
 
 			req.on 'end', =>
+				content = Buffer.concat(chunks).toString()
 				{project_id, file_id} = req.params
 				@files[project_id] ?= {}
-				@files[project_id][file_id] = { content : "test-file-content" }
+				@files[project_id][file_id] = { content }
 				res.sendStatus 200
 
+		app.get "/project/:project_id/file/:file_id", (req, res, next) =>
+			{project_id, file_id} = req.params
+			{ content } = @files[project_id][file_id]
+			res.send content
+
 		app.listen 3009, (error) ->
 			throw error if error?
 		.on "error", (error) ->

services/web/test/acceptance/coffee/helpers/MockURLSource.coffee

@@ -0,0 +1,7 @@
+express = require("express")
+app = express()
+
+module.exports = MockUrlSource =
+	app: app
+	run: (callback) ->
+		app.listen 6543, callback