Merge pull request #124 from overleaf/sk-add-sqlite-heap-limit

Add config for sqlite "soft_heap_limit"

services/git-bridge/Dockerfile

@@ -2,7 +2,7 @@
 
 FROM maven:3-jdk-11 as base
 
-RUN apt-get update && apt-get install -y make git \
+RUN apt-get update && apt-get install -y make git sqlite3 \
  && rm -rf /var/lib/apt/lists
 
 COPY vendor/envsubst /opt/envsubst

services/git-bridge/conf/envsubst_template.json

@@ -28,5 +28,6 @@
     "highGiB": ${GIT_BRIDGE_SWAPJOB_HIGH_GIB:-256},
     "intervalMillis": ${GIT_BRIDGE_SWAPJOB_INTERVAL_MILLIS:-3600000},
     "compressionMethod": "${GIT_BRIDGE_SWAPJOB_COMPRESSION_METHOD:-gzip}"
-  }
+  },
+  "sqliteHeapLimitBytes": ${GIT_BRIDGE_SQLITE_HEAP_LIMIT_BYTES:-0}
 }

services/git-bridge/conf/example_config.json

@@ -28,5 +28,6 @@
         "highGiB": 256,
         "intervalMillis": 3600000,
         "compressionMethod": "gzip"
-    }
+    },
+    "sqliteHeapLimitBytes": 512000000
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java

@@ -33,7 +33,8 @@ public class Config implements JSONSource {
                 Oauth2.asSanitised(config.oauth2),
                 config.repoStore,
                 SwapStoreConfig.sanitisedCopy(config.swapStore),
-                config.swapJob
+                config.swapJob,
+                config.sqliteHeapLimitBytes
         );
     }
 
@@ -52,6 +53,7 @@ public class Config implements JSONSource {
     private SwapStoreConfig swapStore;
     @Nullable
     private SwapJobConfig swapJob;
+    private int sqliteHeapLimitBytes = 0;
 
     public Config(
             String configFilePath
@@ -75,7 +77,8 @@ public class Config implements JSONSource {
             Oauth2 oauth2,
             RepoStoreConfig repoStore,
             SwapStoreConfig swapStore,
-            SwapJobConfig swapJob
+            SwapJobConfig swapJob,
+            int sqliteHeapLimitBytes
     ) {
         this.port = port;
         this.bindIp = bindIp;
@@ -88,6 +91,7 @@ public class Config implements JSONSource {
         this.repoStore = repoStore;
         this.swapStore = swapStore;
         this.swapJob = swapJob;
+        this.sqliteHeapLimitBytes = sqliteHeapLimitBytes;
     }
 
     @Override
@@ -124,6 +128,9 @@ public class Config implements JSONSource {
                 configObject.get("swapJob"),
                 SwapJobConfig.class
         );
+        if (configObject.has("sqliteHeapLimitBytes")) {
+            sqliteHeapLimitBytes = getElement(configObject, "sqliteHeapLimitBytes").getAsInt();
+        }
     }
 
     public String getSanitisedString() {
@@ -146,6 +153,10 @@ public class Config implements JSONSource {
         return rootGitDirectory;
     }
 
+    public int getSqliteHeapLimitBytes() {
+        return this.sqliteHeapLimitBytes;
+    }
+
     public String getAPIBaseURL() {
         return apiBaseURL;
     }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java

@@ -21,8 +21,14 @@ import java.util.stream.Stream;
 public class SqliteDBStore implements DBStore {
 
     private final Connection connection;
+    private int heapLimitBytes = 0;
 
     public SqliteDBStore(File dbFile) {
+        this(dbFile, 0);
+    }
+
+    public SqliteDBStore(File dbFile, int heapLimitBytes) {
+        this.heapLimitBytes = heapLimitBytes;
         try {
             connection = openConnectionTo(dbFile);
             createTables();
@@ -144,6 +150,7 @@ public class SqliteDBStore implements DBStore {
     private void createTables() {
         /* Migrations */
         /* We need to eat exceptions from here */
+        try { doUpdate(new SetSoftHeapLimitPragma(this.heapLimitBytes)); } catch (SQLException ignore) {}
         try { doUpdate(new ProjectsAddLastAccessed()); } catch (SQLException ignore) {}
         try { doUpdate(new ProjectsAddSwapTime()); } catch (SQLException ignore) {}
         try { doUpdate(new ProjectsAddRestoreTime()); } catch (SQLException ignore) {}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/SetSoftHeapLimitPragma.java

@@ -0,0 +1,17 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+public class SetSoftHeapLimitPragma implements SQLUpdate {
+    private int heapLimitBytes = 0;
+
+    public SetSoftHeapLimitPragma(int heapLimitBytes) {
+        this.heapLimitBytes = heapLimitBytes;
+    }
+
+    @Override
+    public String getSQL() {
+        return "PRAGMA soft_heap_limit="+this.heapLimitBytes+";";
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java

@@ -61,7 +61,8 @@ public class GitBridgeServer {
         DBStore dbStore = new SqliteDBStore(
                 Paths.get(
                         repoStore.getRootDirectory().getAbsolutePath()
-                ).resolve(".wlgb").resolve("wlgb.db").toFile()
+                ).resolve(".wlgb").resolve("wlgb.db").toFile(),
+                config.getSqliteHeapLimitBytes()
         );
         SwapStore swapStore = SwapStore.fromConfig(config.getSwapStore());
         SnapshotApi snapshotApi = new NetSnapshotApi();

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java

@@ -94,7 +94,8 @@ public class ConfigTest {
                 "  },\n" +
                 "  \"repoStore\": null,\n" +
                 "  \"swapStore\": null,\n" +
-                "  \"swapJob\": null\n" +
+                "  \"swapJob\": null,\n" +
+                "  \"sqliteHeapLimitBytes\": 0\n" +
                 "}";
         assertEquals(
                 "sanitised config did not hide sensitive fields",

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java

@@ -63,7 +63,8 @@ public class BridgeTest {
                         null,
                         null,
                         null,
-                        null),
+                        null,
+                        0),
                 lock,
                 repoStore,
                 dbStore,