Merge pull request #91 from overleaf/sk-gzip

Gzip Compression in Swap Job

services/git-bridge/conf/example_config.json

@@ -26,6 +26,7 @@
         "minProjects": 50,
         "lowGiB": 128,
         "highGiB": 256,
-        "intervalMillis": 3600000
+        "intervalMillis": 3600000,
+        "compressionMethod": "gzip"
     }
 }

services/git-bridge/conf/local.json

@@ -22,6 +22,7 @@
         "minProjects": 50,
         "lowGiB": 128,
         "highGiB": 256,
-        "intervalMillis": 3600000
+        "intervalMillis": 3600000,
+        "compressionMethod": "gzip"
     }
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java

@@ -24,6 +24,12 @@ public interface DBStore {
 
     String getOldestUnswappedProject();
 
+    void swap(String projectName, String compressionMethod);
+
+    void restore(String projectName);
+
+    String getSwapCompression(String projectName);
+
     int getNumUnswappedProjects();
 
     ProjectState getProjectState(String projectName);

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/noop/NoopDbStore.java

@@ -60,7 +60,16 @@ public class NoopDbStore implements DBStore {
 
     @Override
     public void setLastAccessedTime(String projectName, Timestamp time) {
-
     }
 
+    @Override
+    public void swap(String projectName, String compressionMethod) {}
+
+    @Override
+    public void restore(String projectName) {}
+
+    @Override
+    public String getSwapCompression(String projectName) {
+        return null;
+    }
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java

@@ -104,6 +104,21 @@ public class SqliteDBStore implements DBStore {
         update(new SetProjectLastAccessedTime(projectName, lastAccessed));
     }
 
+    @Override
+    public void swap(String projectName, String compressionMethod) {
+      update(new UpdateSwap(projectName, compressionMethod));
+    }
+
+    @Override
+    public void restore(String projectName) {
+      update(new UpdateRestore(projectName));
+    }
+
+    @Override
+    public String getSwapCompression(String projectName) {
+        return query(new GetSwapCompression(projectName));
+    }
+
     private Connection openConnectionTo(File dbFile) {
         File parentDir = dbFile.getParentFile();
         if (!parentDir.exists() && !parentDir.mkdirs()) {
@@ -127,20 +142,27 @@ public class SqliteDBStore implements DBStore {
     }
 
     private void createTables() {
-        try {
-            doUpdate(new ProjectsAddLastAccessed());
-        } catch (SQLException ignore) {
-            /* We need to eat exceptions from here */
-        }
+        /* Migrations */
+        /* We need to eat exceptions from here */
+        try { doUpdate(new ProjectsAddLastAccessed()); } catch (SQLException ignore) {}
+        try { doUpdate(new ProjectsAddSwapTime()); } catch (SQLException ignore) {}
+        try { doUpdate(new ProjectsAddRestoreTime()); } catch (SQLException ignore) {}
+        try { doUpdate(new ProjectsAddSwapCompression()); } catch (SQLException ignore) {}
+
+        /* Create tables (if they don't exist) */
         Stream.of(
                 new CreateProjectsTableSQLUpdate(),
                 new CreateProjectsIndexLastAccessed(),
                 new CreateURLIndexStoreSQLUpdate(),
                 new CreateIndexURLIndexStore()
         ).forEach(this::update);
+
         /* In the case of needing to change the schema, we need to check that
-           ProjectsAddLastAccessed didn't just fail */
+           migrations didn't just fail */
         Preconditions.checkState(query(new LastAccessedColumnExists()));
+        Preconditions.checkState(query(new SwapTimeColumnExists()));
+        Preconditions.checkState(query(new RestoreTimeColumnExists()));
+        Preconditions.checkState(query(new SwapCompressionColumnExists()));
     }
 
     private void update(SQLUpdate update) {

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetSwapCompression.java

@@ -0,0 +1,39 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class GetSwapCompression implements SQLQuery<String> {
+  private static final String GET_SWAP_COMPRESSION =
+    "SELECT `swap_compression` FROM `projects` WHERE `name` = ?";
+
+  private final String projectName;
+
+  public GetSwapCompression(String projectName) {
+    this.projectName = projectName;
+  }
+
+  @Override
+  public String processResultSet(ResultSet resultSet) throws SQLException {
+    String compression = null;
+    while (resultSet.next()) {
+      compression = resultSet.getString("swap_compression");
+    }
+    return compression;
+  }
+
+  @Override
+  public String getSQL() {
+    return GET_SWAP_COMPRESSION;
+  }
+
+  @Override
+  public void addParametersToStatement(
+    PreparedStatement statement
+  ) throws SQLException {
+    statement.setString(1, projectName);
+  }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/RestoreTimeColumnExists.java

@@ -0,0 +1,26 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class RestoreTimeColumnExists implements SQLQuery<Boolean> {
+    private static final String RESTORE_TIME_COLUMN_EXISTS =
+            "PRAGMA table_info(`projects`)";
+
+    @Override
+    public String getSQL() {
+        return RESTORE_TIME_COLUMN_EXISTS;
+    }
+
+    @Override
+    public Boolean processResultSet(ResultSet resultSet) throws SQLException {
+        while (resultSet.next()) {
+            if (resultSet.getString(2).equals("restore_time")) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapCompressionColumnExists.java

@@ -0,0 +1,27 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class SwapCompressionColumnExists implements SQLQuery<Boolean> {
+    private static final String SWAP_COMPRESSION_COLUMN_EXISTS =
+            "PRAGMA table_info(`projects`)";
+
+    @Override
+    public String getSQL() {
+        return SWAP_COMPRESSION_COLUMN_EXISTS;
+    }
+
+    @Override
+    public Boolean processResultSet(ResultSet resultSet) throws SQLException {
+        while (resultSet.next()) {
+            if (resultSet.getString(2).equals("swap_compression")) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapTimeColumnExists.java

@@ -0,0 +1,27 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class SwapTimeColumnExists implements SQLQuery<Boolean> {
+    private static final String SWAP_TIME_COLUMN_EXISTS =
+            "PRAGMA table_info(`projects`)";
+
+    @Override
+    public String getSQL() {
+        return SWAP_TIME_COLUMN_EXISTS;
+    }
+
+    @Override
+    public Boolean processResultSet(ResultSet resultSet) throws SQLException {
+        while (resultSet.next()) {
+            if (resultSet.getString(2).equals("swap_time")) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddRestoreTime.java

@@ -0,0 +1,14 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+public class ProjectsAddRestoreTime implements SQLUpdate {
+  private static final String PROJECTS_ADD_RESTORE_TIME =
+    "ALTER TABLE `projects`\n" +
+      "ADD COLUMN `restore_time` DATETIME NULL;\n";
+
+  @Override
+  public String getSQL() {
+    return PROJECTS_ADD_RESTORE_TIME;
+  }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapCompression.java

@@ -0,0 +1,14 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+public class ProjectsAddSwapCompression implements SQLUpdate {
+  private static final String PROJECTS_ADD_SWAP_COMPRESSION =
+    "ALTER TABLE `projects`\n" +
+      "ADD COLUMN `swap_compression` VARCHAR NULL;\n";
+
+  @Override
+  public String getSQL() {
+    return PROJECTS_ADD_SWAP_COMPRESSION;
+  }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapTime.java

@@ -0,0 +1,15 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+public class ProjectsAddSwapTime implements SQLUpdate {
+  private static final String PROJECTS_ADD_SWAP_TIME =
+    "ALTER TABLE `projects`\n" +
+      "ADD COLUMN `swap_time` DATETIME NULL;\n";
+
+  @Override
+  public String getSQL() {
+    return PROJECTS_ADD_SWAP_TIME;
+  }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java

@@ -12,6 +12,9 @@ public class CreateProjectsTableSQLUpdate implements SQLUpdate {
             "    `name` VARCHAR NOT NULL DEFAULT '',\n" +
             "    `version_id` INT NOT NULL DEFAULT 0,\n" +
             "    `last_accessed` DATETIME NULL DEFAULT 0,\n" +
+            "    `swap_time` DATETIME NULL,\n" +
+            "    `restore_time` DATETIME NULL,\n" +
+            "    `swap_compression` VARCHAR NULL,\n" +
             "    PRIMARY KEY (`name`)\n" +
             ")";
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateRestore.java

@@ -0,0 +1,40 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
+
+public class UpdateRestore implements SQLUpdate {
+  private static final String UPDATE_RESTORE =
+    "UPDATE `projects`\n" +
+      "SET `last_accessed` = ?,\n" +
+      "    `swap_time` = NULL,\n" +
+      "    `restore_time` = ?,\n" +
+      "    `swap_compression` = NULL\n" +
+      "WHERE `name` = ?;\n";
+
+  private final String projectName;
+  private final Timestamp now;
+
+  public UpdateRestore(String projectName) {
+    this.projectName = projectName;
+    this.now = Timestamp.valueOf(LocalDateTime.now());
+  }
+
+  @Override
+  public String getSQL() {
+    return UPDATE_RESTORE;
+  }
+
+  @Override
+  public void addParametersToStatement(
+    PreparedStatement statement
+  ) throws SQLException {
+    statement.setTimestamp(1, now);
+    statement.setTimestamp(2, now);
+    statement.setString(3, projectName);
+  }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateSwap.java

@@ -0,0 +1,42 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
+
+public class UpdateSwap implements SQLUpdate {
+  private static final String UPDATE_SWAP =
+    "UPDATE `projects`\n" +
+      "SET `last_accessed` = NULL,\n" +
+      "    `swap_time` = ?,\n" +
+      "    `restore_time` = NULL,\n" +
+      "    `swap_compression` = ?\n" +
+      "WHERE `name` = ?;\n";
+
+  private final String projectName;
+  private final String compression;
+  private final Timestamp now;
+
+  public UpdateSwap(String projectName, String compression) {
+    this.projectName = projectName;
+    this.compression = compression;
+    this.now = Timestamp.valueOf(LocalDateTime.now());
+  }
+
+  @Override
+  public String getSQL() {
+    return UPDATE_SWAP;
+  }
+
+  @Override
+  public void addParametersToStatement(
+    PreparedStatement statement
+  ) throws SQLException {
+    statement.setTimestamp(1, now);
+    statement.setString(2, compression);
+    statement.setString(3, projectName);
+  }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java

@@ -136,9 +136,20 @@ public class FSGitRepoStore implements RepoStore {
             long[] sizePtr
     ) throws IOException {
         Project.checkValidProjectName(projectName);
+        Log.info("[{}] bzip2 project", projectName);
         return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr);
     }
 
+    @Override
+    public InputStream gzipProject(
+        String projectName,
+        long[] sizePtr
+    ) throws IOException {
+        Project.checkValidProjectName(projectName);
+        Log.info("[{}] gzip project", projectName);
+        return Tar.gzip.zip(getDotGitForProject(projectName), sizePtr);
+    }
+
     @Override
     public void gcProject(String projectName) throws IOException {
         Project.checkValidProjectName(projectName);
@@ -168,9 +179,30 @@ public class FSGitRepoStore implements RepoStore {
                         "evicted project already exist",
                 projectName
         );
+        Log.info("[{}] un-bzip2 project", projectName);
         Tar.bz2.unzip(dataStream, getDirForProject(projectName));
     }
 
+    @Override
+    public void ungzipProject(
+      String projectName,
+      InputStream dataStream
+    ) throws IOException {
+        Preconditions.checkArgument(
+            Project.isValidProjectName(projectName),
+            "[%s] invalid project name: ",
+            projectName
+        );
+        Preconditions.checkState(
+            getDirForProject(projectName).mkdirs(),
+          "[%s] directories for " +
+            "evicted project already exist",
+            projectName
+        );
+        Log.info("[{}] un-gzip project", projectName);
+        Tar.gzip.unzip(dataStream, getDirForProject(projectName));
+    }
+
     private File getDirForProject(String projectName) {
         Project.checkValidProjectName(projectName);
         return Paths.get(

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java

@@ -50,6 +50,22 @@ public interface RepoStore {
         return bzip2Project(projectName, null);
     }
 
+    /**
+     * Tars and gzips the .git directory of the given project. Throws an
+     * IOException if the project doesn't exist. The returned stream is a copy
+     * of the original .git directory, which must be deleted using remove().
+     */
+    InputStream gzipProject(
+        String projectName,
+        long[] sizePtr
+    ) throws IOException;
+
+    default InputStream gzipProject(
+        String projectName
+    ) throws IOException {
+        return gzipProject(projectName, null);
+    }
+
     void gcProject(String projectName) throws IOException;
 
     /**
@@ -73,4 +89,16 @@ public interface RepoStore {
             InputStream dataStream
     ) throws IOException;
 
+    /**
+     * Ungzips the given data stream into a .git directory for projectName.
+     * Creates the project's git directory.
+     * If projectName already exists, throws an IOException.
+     * @param projectName the name of the project, e.g. abc123
+     * @param dataStream the data stream containing the gzip contents.
+     */
+    void ungzipProject(
+        String projectName,
+        InputStream dataStream
+    ) throws IOException;
+
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java

@@ -13,6 +13,46 @@ import java.util.Optional;
  */
 public interface SwapJob {
 
+    enum CompressionMethod { Bzip2, Gzip }
+
+    static CompressionMethod stringToCompressionMethod(String compressionString) {
+        if (compressionString == null) {
+            return null;
+        }
+        CompressionMethod result;
+        switch (compressionString) {
+            case "gzip":
+                result = CompressionMethod.Gzip;
+                break;
+            case "bzip2":
+                result = CompressionMethod.Bzip2;
+                break;
+            default:
+                result = null;
+                break;
+        }
+        return result;
+    }
+
+    static String compressionMethodAsString(CompressionMethod compressionMethod) {
+        if (compressionMethod == null) {
+            return null;
+        }
+        String result;
+        switch (compressionMethod) {
+            case Gzip:
+                result =  "gzip";
+                break;
+            case Bzip2:
+                result =  "bzip2";
+                break;
+            default:
+                result =  null;
+                break;
+        }
+        return result;
+    }
+
     static SwapJob fromConfig(
             Optional<SwapJobConfig> cfg,
             ProjectLock lock,

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java

@@ -1,5 +1,8 @@
 package uk.ac.ic.wlgitbridge.bridge.swap.job;
 
+import uk.ac.ic.wlgitbridge.util.Log;
+import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob.CompressionMethod;
+
 /**
  * Created by winston on 23/08/2016.
  */
@@ -9,17 +12,20 @@ public class SwapJobConfig {
     private final int lowGiB;
     private final int highGiB;
     private final long intervalMillis;
+    private final String compressionMethod;
 
     public SwapJobConfig(
             int minProjects,
             int lowGiB,
             int highGiB,
-            long intervalMillis
+            long intervalMillis,
+            String compressionMethod
     ) {
         this.minProjects = minProjects;
         this.lowGiB = lowGiB;
         this.highGiB = highGiB;
         this.intervalMillis = intervalMillis;
+        this.compressionMethod = compressionMethod;
     }
 
     public int getMinProjects() {
@@ -38,4 +44,12 @@ public class SwapJobConfig {
         return intervalMillis;
     }
 
+    public SwapJob.CompressionMethod getCompressionMethod() {
+      CompressionMethod result = SwapJob.stringToCompressionMethod(compressionMethod);
+      if (result == null) {
+        Log.info("SwapJobConfig: un-supported compressionMethod '{}', default to 'bzip2'", compressionMethod);
+        result = CompressionMethod.Bzip2;
+      }
+      return result;
+    }
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java

@@ -34,6 +34,7 @@ public class SwapJobImpl implements SwapJob {
     private final RepoStore repoStore;
     private final DBStore dbStore;
     private final SwapStore swapStore;
+    private final CompressionMethod compressionMethod;
 
     private final Timer timer;
 
@@ -51,6 +52,7 @@ public class SwapJobImpl implements SwapJob {
                 GiB * cfg.getLowGiB(),
                 GiB * cfg.getHighGiB(),
                 Duration.ofMillis(cfg.getIntervalMillis()),
+                cfg.getCompressionMethod(),
                 lock,
                 repoStore,
                 dbStore,
@@ -63,6 +65,7 @@ public class SwapJobImpl implements SwapJob {
             long lowWatermarkBytes,
             long highWatermarkBytes,
             Duration interval,
+            CompressionMethod method,
             ProjectLock lock,
             RepoStore repoStore,
             DBStore dbStore,
@@ -72,6 +75,7 @@ public class SwapJobImpl implements SwapJob {
         this.lowWatermarkBytes = lowWatermarkBytes;
         this.highWatermarkBytes = highWatermarkBytes;
         this.interval = interval;
+        this.compressionMethod = method;
         this.lock = lock;
         this.repoStore = repoStore;
         this.dbStore = dbStore;
@@ -196,15 +200,29 @@ public class SwapJobImpl implements SwapJob {
                 Log.error("[{}] Exception while running gc on project: {}", projName, e);
             }
             long[] sizePtr = new long[1];
-            try (InputStream blob = repoStore.bzip2Project(projName, sizePtr)) {
+            try (InputStream blob = getBlobStream(projName, sizePtr)) {
                 swapStore.upload(projName, blob, sizePtr[0]);
-                dbStore.setLastAccessedTime(projName, null);
+                String compression = SwapJob.compressionMethodAsString(compressionMethod);
+                if (compression == null) {
+                  throw new RuntimeException("invalid compression method, should not happen");
+                }
+                dbStore.swap(projName, compression);
                 repoStore.remove(projName);
             }
         }
         Log.info("Evicted project: {}", projName);
     }
 
+    private InputStream getBlobStream(String projName, long[] sizePtr) throws IOException {
+        if (compressionMethod == CompressionMethod.Gzip) {
+          return repoStore.gzipProject(projName, sizePtr);
+        } else if (compressionMethod == CompressionMethod.Bzip2) {
+          return repoStore.bzip2Project(projName, sizePtr);
+        } else {
+          throw new RuntimeException("invalid compression method, should not happen");
+        }
+    }
+
     /**
      * @see SwapJob#restore(String) for high-level description.
      *
@@ -220,15 +238,23 @@ public class SwapJobImpl implements SwapJob {
     public void restore(String projName) throws IOException {
         try (LockGuard __ = lock.lockGuard(projName)) {
             try (InputStream zipped = swapStore.openDownloadStream(projName)) {
-                repoStore.unbzip2Project(
-                        projName,
-                        zipped
-                );
+                String compression = dbStore.getSwapCompression(projName);
+                if (compression == null) {
+                    throw new RuntimeException("Missing compression method during restore, should not happen");
+                }
+                if ("gzip".equals(compression)) {
+                  repoStore.ungzipProject(
+                    projName,
+                    zipped
+                  );
+                } else if ("bzip2".equals(compression)) {
+                  repoStore.unbzip2Project(
+                    projName,
+                    zipped
+                  );
+                }
                 swapStore.remove(projName);
-                dbStore.setLastAccessedTime(
-                        projName,
-                        Timestamp.valueOf(LocalDateTime.now())
-                );
+                dbStore.restore(projName);
             }
         }
     }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java

@@ -6,6 +6,8 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
 import org.apache.commons.compress.utils.IOUtils;
 import org.apache.commons.io.FileUtils;
 
@@ -23,6 +25,44 @@ import java.nio.file.Paths;
  * Caller is responsible for all resources.
  */
 public class Tar {
+    public static class gzip {
+
+        public static InputStream zip(
+            File fileOrDir
+        ) throws IOException {
+            return zip(fileOrDir, null);
+        }
+
+        public static InputStream zip(
+            File fileOrDir,
+            long[] sizePtr
+        ) throws IOException {
+            File tmp = File.createTempFile(fileOrDir.getName(), ".tar.gz");
+            tmp.deleteOnExit();
+            OutputStream target = new FileOutputStream(tmp);
+            /* Closes target */
+            try (OutputStream gz = new GzipCompressorOutputStream(target)) {
+                tarTo(fileOrDir, gz);
+            } catch (IOException e) {
+                tmp.delete();
+                throw e;
+            }
+            if (sizePtr != null) {
+                sizePtr[0] = tmp.length();
+            }
+            return new DeletingFileInputStream(tmp);
+        }
+
+        public static void unzip(
+          InputStream targz,
+          File parentDir
+        ) throws IOException {
+            /* GzipCompressorInputStream does not need closing
+               Closing it would close targz which we should not do */
+            InputStream tar = new GzipCompressorInputStream(targz);
+            untar(tar, parentDir);
+        }
+    }
 
     public static class bz2 {
 

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java

@@ -667,7 +667,7 @@ public class WLGitBridgeIntegrationTest {
         server.start();
         server.setState(states.get("wlgbCanSwapProjects").get("state"));
         wlgb = new GitBridgeApp(new String[] {
-                makeConfigFile(33874, 3874, new SwapJobConfig(1, 0, 0, 250))
+                makeConfigFile(33874, 3874, new SwapJobConfig(1, 0, 0, 250, null))
         });
         wlgb.run();
         File rootGitDir = new File(wlgb.config.getRootGitDirectory());

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java

@@ -69,10 +69,23 @@ public class SqliteDBStoreTest {
         assertEquals("asdf", dbStore.getOldestUnswappedProject());
     }
 
+    @Test
+    public void swapAndRestore() {
+      String projectName = "something";
+      String compression = "bzip2";
+      dbStore.setLatestVersionForProject(projectName, 42);
+      dbStore.swap(projectName, compression);
+      assertNull(dbStore.getOldestUnswappedProject());
+      assertEquals(dbStore.getSwapCompression(projectName), compression);
+      // and restore
+      dbStore.restore(projectName);
+      assertEquals(dbStore.getSwapCompression(projectName), null);
+    }
+
     @Test
     public void noOldestProjectIfAllEvicted() {
         dbStore.setLatestVersionForProject("older", 3);
-        dbStore.setLastAccessedTime("older", null);
+        dbStore.swap("older", "bzip2");
         assertNull(dbStore.getOldestUnswappedProject());
     }
 
@@ -93,7 +106,7 @@ public class SqliteDBStoreTest {
                 )
         );
         assertEquals("older", dbStore.getOldestUnswappedProject());
-        dbStore.setLastAccessedTime("older", null);
+        dbStore.swap("older", "bzip2");
         assertEquals("newer", dbStore.getOldestUnswappedProject());
     }
 
@@ -115,9 +128,9 @@ public class SqliteDBStoreTest {
                 Timestamp.valueOf(LocalDateTime.now())
         );
         assertEquals(1, dbStore.getNumUnswappedProjects());
-        dbStore.setLastAccessedTime(
+        dbStore.swap(
                 "asdf",
-                null
+                "bzip2"
         );
         assertEquals(0, dbStore.getNumUnswappedProjects());
     }
@@ -143,8 +156,8 @@ public class SqliteDBStoreTest {
     @Test
     public void projectStateIsSwappedIfLastAccessedIsNull() {
         dbStore.setLatestVersionForProject("asdf", 1);
-        dbStore.setLastAccessedTime("asdf", null);
+        dbStore.swap("asdf", "bzip2");
         assertEquals(ProjectState.SWAPPED, dbStore.getProjectState("asdf"));
     }
 
-}
+}

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java

@@ -68,6 +68,7 @@ public class SwapJobImplTest {
                 15000,
                 30000,
                 Duration.ofMillis(100),
+                SwapJob.CompressionMethod.Bzip2,
                 lock,
                 repoStore,
                 dbStore,
@@ -82,13 +83,17 @@ public class SwapJobImplTest {
         }
     }
 
+    private void waitASecond() {
+        try { Thread.sleep(1 * 1000); } catch (Exception _e) {}
+    }
+
     @Test
     public void startingTimerAlwaysCausesASwap() {
         swapJob.lowWatermarkBytes = 16384;
         swapJob.interval = Duration.ofHours(1);
         assertEquals(0, swapJob.swaps.get());
         swapJob.start();
-        while (swapJob.swaps.get() <= 0);
+        do { waitASecond(); } while (swapJob.swaps.get() <= 0);
         assertTrue(swapJob.swaps.get() > 0);
     }
 
@@ -97,7 +102,7 @@ public class SwapJobImplTest {
         swapJob.lowWatermarkBytes = 16384;
         assertEquals(0, swapJob.swaps.get());
         swapJob.start();
-        while (swapJob.swaps.get() <= 1);
+        do { waitASecond(); } while (swapJob.swaps.get() <= 1);
         assertTrue(swapJob.swaps.get() > 1);
     }
 
@@ -106,7 +111,7 @@ public class SwapJobImplTest {
         swapJob.highWatermarkBytes = 65536;
         assertEquals(2, dbStore.getNumUnswappedProjects());
         swapJob.start();
-        while (swapJob.swaps.get() < 1);
+        do { waitASecond(); } while (swapJob.swaps.get() < 1);
         assertEquals(2, dbStore.getNumUnswappedProjects());
     }
 
@@ -117,14 +122,45 @@ public class SwapJobImplTest {
         assertEquals(2, dbStore.getNumUnswappedProjects());
         assertEquals("proj2", dbStore.getOldestUnswappedProject());
         swapJob.start();
-        while (swapJob.swaps.get() < 1);
+        do { waitASecond(); } while (swapJob.swaps.get() < 1);
+        assertEquals(1, dbStore.getNumUnswappedProjects());
+        assertEquals("proj1", dbStore.getOldestUnswappedProject());
+        assertEquals("bzip2", dbStore.getSwapCompression("proj2"));
+        swapJob.restore("proj2");
+        assertEquals(null, dbStore.getSwapCompression("proj2"));
+        int numSwaps = swapJob.swaps.get();
+        do { waitASecond(); } while (swapJob.swaps.get() <= numSwaps);
+        assertEquals(1, dbStore.getNumUnswappedProjects());
+        assertEquals("proj2", dbStore.getOldestUnswappedProject());
+    }
+
+    @Test
+    public void swapCompressionGzip() throws IOException {
+        swapJob = new SwapJobImpl(
+                1,
+                15000,
+                30000,
+                Duration.ofMillis(100),
+                SwapJob.CompressionMethod.Gzip,
+                lock,
+                repoStore,
+                dbStore,
+                swapStore
+        );
+        swapJob.lowWatermarkBytes = 16384;
+        assertEquals(2, dbStore.getNumUnswappedProjects());
+        assertEquals("proj2", dbStore.getOldestUnswappedProject());
+        swapJob.start();
+        do { waitASecond(); } while (swapJob.swaps.get() < 1);
         assertEquals(1, dbStore.getNumUnswappedProjects());
         assertEquals("proj1", dbStore.getOldestUnswappedProject());
+        assertEquals("gzip", dbStore.getSwapCompression("proj2"));
         swapJob.restore("proj2");
+        assertEquals(null, dbStore.getSwapCompression("proj2"));
         int numSwaps = swapJob.swaps.get();
-        while (swapJob.swaps.get() <= numSwaps);
+        do { waitASecond(); } while (swapJob.swaps.get() <= numSwaps);
         assertEquals(1, dbStore.getNumUnswappedProjects());
         assertEquals("proj2", dbStore.getOldestUnswappedProject());
     }
 
-}
+}