|
@@ -6,6 +6,15 @@ fs = require("fs")
|
|
|
logger = require "logger-sharelatex"
|
|
logger = require "logger-sharelatex"
|
|
|
async = require "async"
|
|
async = require "async"
|
|
|
|
|
|
|
|
|
|
+queue = async.queue((task, cb)->
|
|
|
|
|
+ console.log("running task")
|
|
|
|
|
+ task(cb)
|
|
|
|
|
+, 1)
|
|
|
|
|
+
|
|
|
|
|
+console.log("hi there queue")
|
|
|
|
|
+queue.drain = ()->
|
|
|
|
|
+ console.log('HI all items have been processed')
|
|
|
|
|
+
|
|
|
module.exports = UrlCache =
|
|
module.exports = UrlCache =
|
|
|
downloadUrlToFile: (project_id, url, destPath, lastModified, callback = (error) ->) ->
|
|
downloadUrlToFile: (project_id, url, destPath, lastModified, callback = (error) ->) ->
|
|
|
UrlCache._ensureUrlIsInCache project_id, url, lastModified, (error, pathToCachedUrl) =>
|
|
UrlCache._ensureUrlIsInCache project_id, url, lastModified, (error, pathToCachedUrl) =>
|
|
@@ -51,8 +60,9 @@ module.exports = UrlCache =
|
|
|
_doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) ->
|
|
_doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) ->
|
|
|
if !lastModified?
|
|
if !lastModified?
|
|
|
return callback null, true
|
|
return callback null, true
|
|
|
-
|
|
|
|
|
|
|
+ console.log "about to get _findUrlDetails"
|
|
|
UrlCache._findUrlDetails project_id, url, (error, urlDetails) ->
|
|
UrlCache._findUrlDetails project_id, url, (error, urlDetails) ->
|
|
|
|
|
+ console.log error, urlDetails, "_findUrlDetails result"
|
|
|
return callback(error) if error?
|
|
return callback(error) if error?
|
|
|
if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime()
|
|
if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime()
|
|
|
return callback null, true
|
|
return callback null, true
|
|
@@ -94,36 +104,44 @@ module.exports = UrlCache =
|
|
|
return callback()
|
|
return callback()
|
|
|
|
|
|
|
|
_findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) ->
|
|
_findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) ->
|
|
|
- console.log("_findUrlDetails")
|
|
|
|
|
- db.UrlCache.find(where: { url: url, project_id: project_id })
|
|
|
|
|
- .then((urlDetails) -> callback null, urlDetails)
|
|
|
|
|
- .error callback
|
|
|
|
|
|
|
+ job = (cb)->
|
|
|
|
|
+ db.UrlCache.find(where: { url: url, project_id: project_id })
|
|
|
|
|
+ .then((urlDetails) -> cb null, urlDetails)
|
|
|
|
|
+ .error cb
|
|
|
|
|
+ queue.push job, callback
|
|
|
|
|
|
|
|
_updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) ->
|
|
_updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) ->
|
|
|
- console.log("_updateOrCreateUrlDetails")
|
|
|
|
|
- db.UrlCache.findOrCreate(where: {url: url, project_id: project_id})
|
|
|
|
|
- .spread(
|
|
|
|
|
- (urlDetails, created) ->
|
|
|
|
|
- urlDetails.updateAttributes(lastModified: lastModified)
|
|
|
|
|
- .then(() -> callback())
|
|
|
|
|
- .error(callback)
|
|
|
|
|
- )
|
|
|
|
|
- .error callback
|
|
|
|
|
|
|
+ job = (cb)->
|
|
|
|
|
+ console.log("_updateOrCreateUrlDetails")
|
|
|
|
|
+ db.UrlCache.findOrCreate(where: {url: url, project_id: project_id})
|
|
|
|
|
+ .spread(
|
|
|
|
|
+ (urlDetails, created) ->
|
|
|
|
|
+ urlDetails.updateAttributes(lastModified: lastModified)
|
|
|
|
|
+ .then(() -> cb())
|
|
|
|
|
+ .error(cb)
|
|
|
|
|
+ )
|
|
|
|
|
+ .error cb
|
|
|
|
|
+ queue.push(job, callback)
|
|
|
|
|
|
|
|
_clearUrlDetails: (project_id, url, callback = (error) ->) ->
|
|
_clearUrlDetails: (project_id, url, callback = (error) ->) ->
|
|
|
- console.log("_clearUrlDetails")
|
|
|
|
|
- db.UrlCache.destroy(where: {url: url, project_id: project_id})
|
|
|
|
|
- .then(() -> callback null)
|
|
|
|
|
- .error callback
|
|
|
|
|
|
|
+ job = (cb)->
|
|
|
|
|
+ console.log("_clearUrlDetails")
|
|
|
|
|
+ db.UrlCache.destroy(where: {url: url, project_id: project_id})
|
|
|
|
|
+ .then(() -> cb null)
|
|
|
|
|
+ .error cb
|
|
|
|
|
+ queue.push(job, callback)
|
|
|
|
|
+
|
|
|
|
|
|
|
|
_findAllUrlsInProject: (project_id, callback = (error, urls) ->) ->
|
|
_findAllUrlsInProject: (project_id, callback = (error, urls) ->) ->
|
|
|
- console.log("_findAllUrlsInProject")
|
|
|
|
|
- db.UrlCache.findAll(where: { project_id: project_id })
|
|
|
|
|
- .then(
|
|
|
|
|
- (urlEntries) ->
|
|
|
|
|
- callback null, urlEntries.map((entry) -> entry.url)
|
|
|
|
|
- )
|
|
|
|
|
- .error callback
|
|
|
|
|
|
|
+ job = (cb)->
|
|
|
|
|
+ console.log("_findAllUrlsInProject")
|
|
|
|
|
+ db.UrlCache.findAll(where: { project_id: project_id })
|
|
|
|
|
+ .then(
|
|
|
|
|
+ (urlEntries) ->
|
|
|
|
|
+ cb null, urlEntries.map((entry) -> entry.url)
|
|
|
|
|
+ )
|
|
|
|
|
+ .error cb
|
|
|
|
|
+ queue.push(job, callback)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|