Merge branch 'master' into release

.gitignore

@@ -12,6 +12,7 @@ tags
 chat
 spelling
 real-time
+migrations/*.js
 
 data
 tmp

CHANGELOG.md

@@ -0,0 +1,11 @@
+v0.1.2
+------
+
+* Re-brand open-source ShareLaTeX code as 'ShareLaTeX Community Edition'.
+* The Dropbox and template code has been extracted out into a separate module and removed from the ShareLaTeX Community Edition. There should be no broken features due to lack of open source components now.
+* Websockets and real-time data now go to a separate light-weight [real-time](https://github.com/sharelatex/real-time) service.
+* Updated PDF viewer that loads page-by-page for much quicker loading times on large documents.
+* Links are clickable in chat messages.
+* Mathjax libraries are now served locally.
+* Optimisation of Angular digest loop in editor should reduce CPU usage in certain cases.
+* Numerous small bug fixes.

Gruntfile.coffee

@@ -7,6 +7,8 @@ semver = require "semver"
 knox = require "knox"
 crypto = require "crypto"
 async = require "async"
+settings = require("settings-sharelatex")
+
 
 SERVICES = [{
 	name: "web"
@@ -56,6 +58,7 @@ module.exports = (grunt) ->
 	grunt.loadNpmTasks 'grunt-available-tasks'
 	grunt.loadNpmTasks 'grunt-concurrent'
 	grunt.loadNpmTasks "grunt-contrib-coffee"
+	grunt.loadNpmTasks "grunt-shell"
 
 
 	execute = {}
@@ -83,7 +86,9 @@ module.exports = (grunt) ->
 				options:
 					bare:true
 
-
+		shell:
+			migrate:
+				command: "./node_modules/east/bin/east migrate --adapter east-mongo --url #{settings.mongo.url}"
 
 		availabletasks:
 			tasks:
@@ -162,7 +167,7 @@ module.exports = (grunt) ->
 		Helpers.buildUpstartScripts()
 
 
-	grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate']
+	#grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate']
 
 
 	Helpers =

README.md

@@ -15,6 +15,11 @@ We have detailed installation instructions in our wiki:
 
 **If you have any problems, have a look at our page of [Frequent Problems and Questions](https://github.com/sharelatex/sharelatex/wiki/FAQ).**
 
+Upgrading
+---------
+
+If you are upgrading from a previous version of ShareLaTeX, please see the [Release Notes section on the Wiki] (https://github.com/sharelatex/sharelatex/wiki/Home) for all of the versions between your current version and the version you are upgrading to.
+
 
 Dependencies
 ------------

config/settings.development.coffee.example

@@ -93,6 +93,14 @@ module.exports =
 	# when emails are sent out and in generated links:
 	siteUrl : 'http://localhost:3000'
 	
+	# The websocket layer of ShareLaTeX runs as separate service.
+	# When running locally or in development, you can point the client to this
+	# service directly on port 3026. If you are running behind a reverse proxy (Nginx, etc)
+	# then websocketsUrl should be the same as siteUrl, with your reverse
+	# proxy responible for sending websocket traffic to the websocket service
+	# rather than connecting directly.
+	websocketsUrl: 'http://localhost:3026'
+	
 	# If provided, a sessionSecret is used to sign cookies so that they cannot be
 	# spoofed. This is recommended.
 	security:

migrations/1422460849371-doc-lines.coffee → migrations/1_doc_lines.coffee

@@ -1,31 +1,32 @@
-
+Settings = require "settings-sharelatex"
 fs = require("fs")
 mongojs = require("mongojs")
 ObjectId = mongojs.ObjectId
-db = mongojs('sharelatex', ['projects', 'docs'])
+console.log Settings.mongo.url
+db = mongojs(Settings.mongo.url, ['projects', 'docs'])
 _ = require("lodash")
 async = require("async")
 exec = require("child_process").exec
 
 finished_projects_path = "/tmp/finished-projects"
 all_projects_path = "/tmp/all-projects"
+project_too_large_path = "/tmp/large_projects"
+
 
 printProgress = ->
 	exec "wc #{finished_projects_path}", (error, results) ->
-		#console.log results
 		setTimeout printProgress, 1000 * 30
 
 checkIfFileHasBeenProccessed = (project_id, callback)->
 	exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
 		hasBeenProcessed = _.include(results, project_id)
-		#console.log hasBeenProcessed, project_id
-		callback(null, hasBeenProcessed)
+		callback(error, hasBeenProcessed)
 
 loadProjectIds = (callback)->
+	console.log "loading project ids from #{all_projects_path}"
 	fs.readFile all_projects_path, "utf-8", (err, data)->
-		console.log data.length
 		ids = data.split("\n")
-		console.log ids.length
+		console.log "loaded #{ids.length} project ids from #{all_projects_path}"
 		callback err, ids
 
 getAndWriteProjectids = (callback)->
@@ -38,9 +39,14 @@ getAndWriteProjectids = (callback)->
 		fs.writeFile all_projects_path, fileData, ->
 			callback(err, ids)
 
+markProjectAsToLargeAndFinished = (project_id, callback)->
+	console.log "#{project_id} too large"
+	markProjectAsProcessed project_id, (err)->
+		fs.appendFile project_too_large_path, "#{project_id}\n", callback
+
 getProjectIds = (callback)->
 	exists = fs.existsSync all_projects_path
-	if exists 
+	if exists
 		loadProjectIds callback
 	else
 		getAndWriteProjectids callback
@@ -52,8 +58,11 @@ getAllDocs = (project_id, callback = (error, docs) ->) ->
 	db.projects.findOne _id:ObjectId(project_id), (error, project) ->
 		return callback(error) if error?
 		if !project?
-			console.error("No such project: #{project_id}")
-			return callback("no such project #{project_id}")
+			console.log "no such project #{project_id}"
+			return callback()
+		size = require("../node_modules/mongojs/node_modules/mongodb/node_modules/bson/").BSONPure.BSON.calculateObjectSize(project)
+		if size > 12000000 #12mb
+			return markProjectAsToLargeAndFinished project_id, callback
 		findAllDocsInProject project, (error, docs) ->
 			return callback(error) if error?
 			return callback null, docs
@@ -82,17 +91,33 @@ _findAllDocsInFolder = (folder = {}) ->
 	return docs
 
 insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)->
+	if !project_id?
+		return callback("no project id")
+	if !doc_id?
+		return callback()
+	if !lines?
+		lines = [""]
 	update = {}
-	update["_id"] = ObjectId(doc_id)
+	update["_id"] = ObjectId(doc_id.toString())
 	update["lines"] = lines
 	update["project_id"] = ObjectId(project_id)
-	update["rev"] = oldRev
-	db.docs.insert _id: ObjectId(doc_id), callback
+	update["rev"] = oldRev || 0
+	db.docs.insert update, callback
 
 saveDocsIntoMongo = (project_id, docs, callback)->
 	jobs = _.map docs, (doc)->
 		(cb)->
-			insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb
+			if !doc?
+				console.error "null doc in project #{project_id}" #just skip it, not a big deal
+				return cb()
+			insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)->
+				if err?.code == 11000 #duplicate key, doc already in there so its not a problem.
+					err = undefined
+				if err?
+					console.log "error inserting doc into doc collection", err
+				cb(err)
+
+
 	async.series jobs, callback
 
 
@@ -101,33 +126,36 @@ processNext = (project_id, callback)->
 		if hasBeenProcessed
 			console.log "#{project_id} already procssed, skipping"
 			return callback()
+		console.log "#{project_id} processing"
 		getAllDocs project_id, (err, docs)->
 			if err?
 				console.error err, project_id, "could not get all docs"
-				return callback()
-			saveDocsIntoMongo project_id, docs, ->
-				if err?
-					console.error err, project_id, "could not save docs into mongo"
-					return callback()
-				markProjectAsProcessed project_id, ->
-					callback()
-
-getProjectIds (err, ids)->
-	printProgress()
-	jobs = _.map ids, (id)->
-		return (cb)->
-			processNext(id, cb)
-	async.series jobs, (err)->
-		if err?
-			console.error err, "at end of jobs"
-		else
-			console.log "finished"
-		process.exit()
-	
-exports.up = (next)->
-
-	next()
+				return callback(err)
+			else
+				saveDocsIntoMongo project_id, docs, (err)->
+					if err?
+						console.error err, project_id, "could not save docs into mongo"
+						return callback(err)
+					markProjectAsProcessed project_id, (err)->
+						setTimeout(
+							-> callback(err)
+						,500)
+
+
+
+exports.migrate = (client, done = ->)->
+	getProjectIds (err, ids)->
+		printProgress()
+		jobs = _.map ids, (id)->
+			return (cb)->
+				processNext(id, cb)
+		async.series jobs, (err)->
+			if err?
+				console.error err, "at end of jobs"
+			else
+				console.log "finished"
+			done(err)
 
 
-exports.down = (next)->
+exports.rollback = (next)->
 	next()

migrations/about_migrations.md

@@ -0,0 +1,2 @@
+* if migration is stopped mid way it will start at the beginging next time
+* to see the run migrations do db.getCollection('_migrations').find() you can't do db._migrations.find()

package.json

@@ -4,6 +4,9 @@
   "description": "An online collaborative LaTeX editor",
   "dependencies": {
     "async": "^0.9.0",
+    "east": "^0.2.3",
+    "east-mongo": "^0.1.2",
+    "grunt-shell": "^1.1.1",
     "lodash": "^3.0.0",
     "mongojs": "^0.18.1",
     "rimraf": "~2.2.6",

package/upstart/sharelatex-real-time

@@ -0,0 +1,28 @@
+description "sharelatex-web"
+author      "ShareLaTeX <team@sharelatex.com>"
+
+start on (local-filesystems and net-device-up IFACE!=lo)
+stop on shutdown
+
+respawn
+
+limit nofile 8192 8192
+
+pre-start script
+    mkdir -p /var/log/sharelatex
+end script
+
+script
+    SERVICE=real-time
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script