From 169de1feade4428d2d2f9ad9d535bd9ff3d63056 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jan 2021 13:51:41 +0000 Subject: [PATCH 01/12] Configure swap compression method: bzip2 or gzip --- services/git-bridge/conf/example_config.json | 3 +- services/git-bridge/conf/local.json | 3 +- .../wlgitbridge/bridge/swap/job/SwapJob.java | 40 +++++++++++++++++++ .../bridge/swap/job/SwapJobConfig.java | 16 +++++++- 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/services/git-bridge/conf/example_config.json b/services/git-bridge/conf/example_config.json index 38b14b9787..c716e998b4 100644 --- a/services/git-bridge/conf/example_config.json +++ b/services/git-bridge/conf/example_config.json @@ -26,6 +26,7 @@ "minProjects": 50, "lowGiB": 128, "highGiB": 256, - "intervalMillis": 3600000 + "intervalMillis": 3600000, + "compressionMethod": "bzip2" } } diff --git a/services/git-bridge/conf/local.json b/services/git-bridge/conf/local.json index 0acb24ac0a..17cc37796f 100644 --- a/services/git-bridge/conf/local.json +++ b/services/git-bridge/conf/local.json @@ -22,6 +22,7 @@ "minProjects": 50, "lowGiB": 128, "highGiB": 256, - "intervalMillis": 3600000 + "intervalMillis": 3600000, + "compressionMethod": "bzip2" } } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java index 05d6fbebb2..8ea19f7d0e 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java @@ -13,6 +13,46 @@ import java.util.Optional; */ public interface SwapJob { + enum CompressionMethod { Bzip2, Gzip } + + static CompressionMethod stringToCompressionMethod(String compressionString) { + if (compressionString == null) { + return null; + } + CompressionMethod result; + switch (compressionString) { + case "gzip": + result = CompressionMethod.Gzip; + break; + case "bzip2": + result = CompressionMethod.Bzip2; + break; + default: + result = null; + break; + } + return result; + } + + static String compressionMethodAsString(CompressionMethod compressionMethod) { + if (compressionMethod == null) { + return null; + } + String result; + switch (compressionMethod) { + case Gzip: + result = "gzip"; + break; + case Bzip2: + result = "bzip2"; + break; + default: + result = null; + break; + } + return result; + } + static SwapJob fromConfig( Optional cfg, ProjectLock lock, diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java index 057cb8148e..312696473d 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java @@ -1,5 +1,8 @@ package uk.ac.ic.wlgitbridge.bridge.swap.job; +import uk.ac.ic.wlgitbridge.util.Log; +import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob.CompressionMethod; + /** * Created by winston on 23/08/2016. */ @@ -9,17 +12,20 @@ public class SwapJobConfig { private final int lowGiB; private final int highGiB; private final long intervalMillis; + private final String compressionMethod; public SwapJobConfig( int minProjects, int lowGiB, int highGiB, - long intervalMillis + long intervalMillis, + String compressionMethod ) { this.minProjects = minProjects; this.lowGiB = lowGiB; this.highGiB = highGiB; this.intervalMillis = intervalMillis; + this.compressionMethod = compressionMethod; } public int getMinProjects() { @@ -38,4 +44,12 @@ public class SwapJobConfig { return intervalMillis; } + public SwapJob.CompressionMethod getCompressionMethod() { + CompressionMethod result = SwapJob.stringToCompressionMethod(compressionMethod); + if (result == null) { + Log.info("SwapJobConfig: un-supported compressionMethod '{}', default to 'bzip2'", compressionMethod); + result = CompressionMethod.Bzip2; + } + return result; + } } From 8496871ab235fb4d2be8cd7c7227b6d5dc9e6cb9 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jan 2021 13:54:01 +0000 Subject: [PATCH 02/12] Add gzip support to Tar helper --- .../java/uk/ac/ic/wlgitbridge/util/Tar.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java index d64d03e50a..6de56e7acf 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java @@ -6,6 +6,8 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.io.FileUtils; @@ -23,6 +25,44 @@ import java.nio.file.Paths; * Caller is responsible for all resources. */ public class Tar { + public static class gzip { + + public static InputStream zip( + File fileOrDir + ) throws IOException { + return zip(fileOrDir, null); + } + + public static InputStream zip( + File fileOrDir, + long[] sizePtr + ) throws IOException { + File tmp = File.createTempFile(fileOrDir.getName(), ".tar.gz"); + tmp.deleteOnExit(); + OutputStream target = new FileOutputStream(tmp); + /* Closes target */ + try (OutputStream gz = new GzipCompressorOutputStream(target)) { + tarTo(fileOrDir, gz); + } catch (IOException e) { + tmp.delete(); + throw e; + } + if (sizePtr != null) { + sizePtr[0] = tmp.length(); + } + return new DeletingFileInputStream(tmp); + } + + public static void unzip( + InputStream targz, + File parentDir + ) throws IOException { + /* GzipCompressorInputStream does not need closing + Closing it would close targz which we should not do */ + InputStream tar = new GzipCompressorInputStream(targz); + untar(tar, parentDir); + } + } public static class bz2 { From 1c81fcf501f1835d7b9e982bd7fd387772f27ca6 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jan 2021 13:52:42 +0000 Subject: [PATCH 03/12] Add gzip to RepoStore interface --- .../ic/wlgitbridge/bridge/repo/RepoStore.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java index 737d065f34..9aba00f477 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java @@ -50,6 +50,22 @@ public interface RepoStore { return bzip2Project(projectName, null); } + /** + * Tars and gzips the .git directory of the given project. Throws an + * IOException if the project doesn't exist. The returned stream is a copy + * of the original .git directory, which must be deleted using remove(). + */ + InputStream gzipProject( + String projectName, + long[] sizePtr + ) throws IOException; + + default InputStream gzipProject( + String projectName + ) throws IOException { + return gzipProject(projectName, null); + } + void gcProject(String projectName) throws IOException; /** @@ -73,4 +89,16 @@ public interface RepoStore { InputStream dataStream ) throws IOException; + /** + * Ungzips the given data stream into a .git directory for projectName. + * Creates the project's git directory. + * If projectName already exists, throws an IOException. + * @param projectName the name of the project, e.g. abc123 + * @param dataStream the data stream containing the gzip contents. + */ + void ungzipProject( + String projectName, + InputStream dataStream + ) throws IOException; + } From eba7cbe696c9772ba99da979ac163d68866d7192 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jan 2021 13:56:19 +0000 Subject: [PATCH 04/12] Implement gzip in FSGitRepoStore --- .../bridge/repo/FSGitRepoStore.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java index 72eb44a5b4..c440f30115 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java @@ -139,6 +139,15 @@ public class FSGitRepoStore implements RepoStore { return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr); } + @Override + public InputStream gzipProject( + String projectName, + long[] sizePtr + ) throws IOException { + Project.checkValidProjectName(projectName); + return Tar.gzip.zip(getDotGitForProject(projectName), sizePtr); + } + @Override public void gcProject(String projectName) throws IOException { Project.checkValidProjectName(projectName); @@ -171,6 +180,25 @@ public class FSGitRepoStore implements RepoStore { Tar.bz2.unzip(dataStream, getDirForProject(projectName)); } + @Override + public void ungzipProject( + String projectName, + InputStream dataStream + ) throws IOException { + Preconditions.checkArgument( + Project.isValidProjectName(projectName), + "[%s] invalid project name: ", + projectName + ); + Preconditions.checkState( + getDirForProject(projectName).mkdirs(), + "[%s] directories for " + + "evicted project already exist", + projectName + ); + Tar.gzip.unzip(dataStream, getDirForProject(projectName)); + } + private File getDirForProject(String projectName) { Project.checkValidProjectName(projectName); return Paths.get( From 21c524d8b60a683c978f622d395a9fecea5932c4 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 21 Jan 2021 09:07:42 +0000 Subject: [PATCH 05/12] Log actions when (un)compressing project --- .../java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java index c440f30115..c323930157 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java @@ -136,6 +136,7 @@ public class FSGitRepoStore implements RepoStore { long[] sizePtr ) throws IOException { Project.checkValidProjectName(projectName); + Log.info("[{}] bzip2 project", projectName); return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr); } @@ -145,6 +146,7 @@ public class FSGitRepoStore implements RepoStore { long[] sizePtr ) throws IOException { Project.checkValidProjectName(projectName); + Log.info("[{}] gzip project", projectName); return Tar.gzip.zip(getDotGitForProject(projectName), sizePtr); } @@ -177,6 +179,7 @@ public class FSGitRepoStore implements RepoStore { "evicted project already exist", projectName ); + Log.info("[{}] un-bzip2 project", projectName); Tar.bz2.unzip(dataStream, getDirForProject(projectName)); } @@ -196,6 +199,7 @@ public class FSGitRepoStore implements RepoStore { "evicted project already exist", projectName ); + Log.info("[{}] un-gzip project", projectName); Tar.gzip.unzip(dataStream, getDirForProject(projectName)); } From 83d9d122bf4325db7b3851a34a5597d8c69d373c Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jan 2021 13:57:34 +0000 Subject: [PATCH 06/12] Add swap/restore methods to DBStore interface --- .../main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java index 0a13b26844..ff1e05ee0c 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java @@ -24,6 +24,12 @@ public interface DBStore { String getOldestUnswappedProject(); + void swap(String projectName, String compressionMethod); + + void restore(String projectName); + + String getSwapCompression(String projectName); + int getNumUnswappedProjects(); ProjectState getProjectState(String projectName); From a3d4976ce2106c4006aab6c77d59c23ad3c3eacf Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jan 2021 13:58:08 +0000 Subject: [PATCH 07/12] Implement swap/restore on NoopDbStore --- .../ac/ic/wlgitbridge/bridge/db/noop/NoopDbStore.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/noop/NoopDbStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/noop/NoopDbStore.java index 9519bf0fe4..0f0f9abb7c 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/noop/NoopDbStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/noop/NoopDbStore.java @@ -60,7 +60,16 @@ public class NoopDbStore implements DBStore { @Override public void setLastAccessedTime(String projectName, Timestamp time) { - } + @Override + public void swap(String projectName, String compressionMethod) {} + + @Override + public void restore(String projectName) {} + + @Override + public String getSwapCompression(String projectName) { + return null; + } } From 961cec5bc8ee7f8d2a295928b3b070708460bf64 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jan 2021 13:59:37 +0000 Subject: [PATCH 08/12] Implement swap/restore in SqliteDBStore --- .../bridge/db/sqlite/SqliteDBStore.java | 34 ++++++++++++--- .../db/sqlite/query/GetSwapCompression.java | 39 +++++++++++++++++ .../sqlite/query/RestoreTimeColumnExists.java | 26 ++++++++++++ .../query/SwapCompressionColumnExists.java | 27 ++++++++++++ .../db/sqlite/query/SwapTimeColumnExists.java | 27 ++++++++++++ .../update/alter/ProjectsAddRestoreTime.java | 14 +++++++ .../alter/ProjectsAddSwapCompression.java | 14 +++++++ .../update/alter/ProjectsAddSwapTime.java | 15 +++++++ .../create/CreateProjectsTableSQLUpdate.java | 3 ++ .../sqlite/update/insert/UpdateRestore.java | 40 ++++++++++++++++++ .../db/sqlite/update/insert/UpdateSwap.java | 42 +++++++++++++++++++ 11 files changed, 275 insertions(+), 6 deletions(-) create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetSwapCompression.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/RestoreTimeColumnExists.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapCompressionColumnExists.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapTimeColumnExists.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddRestoreTime.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapCompression.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapTime.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateRestore.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateSwap.java diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java index 20bf412624..725347bc0d 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java @@ -104,6 +104,21 @@ public class SqliteDBStore implements DBStore { update(new SetProjectLastAccessedTime(projectName, lastAccessed)); } + @Override + public void swap(String projectName, String compressionMethod) { + update(new UpdateSwap(projectName, compressionMethod)); + } + + @Override + public void restore(String projectName) { + update(new UpdateRestore(projectName)); + } + + @Override + public String getSwapCompression(String projectName) { + return query(new GetSwapCompression(projectName)); + } + private Connection openConnectionTo(File dbFile) { File parentDir = dbFile.getParentFile(); if (!parentDir.exists() && !parentDir.mkdirs()) { @@ -127,20 +142,27 @@ public class SqliteDBStore implements DBStore { } private void createTables() { - try { - doUpdate(new ProjectsAddLastAccessed()); - } catch (SQLException ignore) { - /* We need to eat exceptions from here */ - } + /* Migrations */ + /* We need to eat exceptions from here */ + try { doUpdate(new ProjectsAddLastAccessed()); } catch (SQLException ignore) {} + try { doUpdate(new ProjectsAddSwapTime()); } catch (SQLException ignore) {} + try { doUpdate(new ProjectsAddRestoreTime()); } catch (SQLException ignore) {} + try { doUpdate(new ProjectsAddSwapCompression()); } catch (SQLException ignore) {} + + /* Create tables (if they don't exist) */ Stream.of( new CreateProjectsTableSQLUpdate(), new CreateProjectsIndexLastAccessed(), new CreateURLIndexStoreSQLUpdate(), new CreateIndexURLIndexStore() ).forEach(this::update); + /* In the case of needing to change the schema, we need to check that - ProjectsAddLastAccessed didn't just fail */ + migrations didn't just fail */ Preconditions.checkState(query(new LastAccessedColumnExists())); + Preconditions.checkState(query(new SwapTimeColumnExists())); + Preconditions.checkState(query(new RestoreTimeColumnExists())); + Preconditions.checkState(query(new SwapCompressionColumnExists())); } private void update(SQLUpdate update) { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetSwapCompression.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetSwapCompression.java new file mode 100644 index 0000000000..c7f4d5b510 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetSwapCompression.java @@ -0,0 +1,39 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +public class GetSwapCompression implements SQLQuery { + private static final String GET_SWAP_COMPRESSION = + "SELECT `swap_compression` FROM `projects` WHERE `name` = ?"; + + private final String projectName; + + public GetSwapCompression(String projectName) { + this.projectName = projectName; + } + + @Override + public String processResultSet(ResultSet resultSet) throws SQLException { + String compression = null; + while (resultSet.next()) { + compression = resultSet.getString("swap_compression"); + } + return compression; + } + + @Override + public String getSQL() { + return GET_SWAP_COMPRESSION; + } + + @Override + public void addParametersToStatement( + PreparedStatement statement + ) throws SQLException { + statement.setString(1, projectName); + } +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/RestoreTimeColumnExists.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/RestoreTimeColumnExists.java new file mode 100644 index 0000000000..ed4d0567ae --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/RestoreTimeColumnExists.java @@ -0,0 +1,26 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class RestoreTimeColumnExists implements SQLQuery { + private static final String RESTORE_TIME_COLUMN_EXISTS = + "PRAGMA table_info(`projects`)"; + + @Override + public String getSQL() { + return RESTORE_TIME_COLUMN_EXISTS; + } + + @Override + public Boolean processResultSet(ResultSet resultSet) throws SQLException { + while (resultSet.next()) { + if (resultSet.getString(2).equals("restore_time")) { + return true; + } + } + return false; + } +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapCompressionColumnExists.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapCompressionColumnExists.java new file mode 100644 index 0000000000..1b1754f26e --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapCompressionColumnExists.java @@ -0,0 +1,27 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class SwapCompressionColumnExists implements SQLQuery { + private static final String SWAP_COMPRESSION_COLUMN_EXISTS = + "PRAGMA table_info(`projects`)"; + + @Override + public String getSQL() { + return SWAP_COMPRESSION_COLUMN_EXISTS; + } + + @Override + public Boolean processResultSet(ResultSet resultSet) throws SQLException { + while (resultSet.next()) { + if (resultSet.getString(2).equals("swap_compression")) { + return true; + } + } + return false; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapTimeColumnExists.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapTimeColumnExists.java new file mode 100644 index 0000000000..9426dedc5f --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapTimeColumnExists.java @@ -0,0 +1,27 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class SwapTimeColumnExists implements SQLQuery { + private static final String SWAP_TIME_COLUMN_EXISTS = + "PRAGMA table_info(`projects`)"; + + @Override + public String getSQL() { + return SWAP_TIME_COLUMN_EXISTS; + } + + @Override + public Boolean processResultSet(ResultSet resultSet) throws SQLException { + while (resultSet.next()) { + if (resultSet.getString(2).equals("swap_time")) { + return true; + } + } + return false; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddRestoreTime.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddRestoreTime.java new file mode 100644 index 0000000000..dd92efcab5 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddRestoreTime.java @@ -0,0 +1,14 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; + +public class ProjectsAddRestoreTime implements SQLUpdate { + private static final String PROJECTS_ADD_RESTORE_TIME = + "ALTER TABLE `projects`\n" + + "ADD COLUMN `restore_time` DATETIME NULL;\n"; + + @Override + public String getSQL() { + return PROJECTS_ADD_RESTORE_TIME; + } +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapCompression.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapCompression.java new file mode 100644 index 0000000000..c5599d9116 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapCompression.java @@ -0,0 +1,14 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; + +public class ProjectsAddSwapCompression implements SQLUpdate { + private static final String PROJECTS_ADD_SWAP_COMPRESSION = + "ALTER TABLE `projects`\n" + + "ADD COLUMN `swap_compression` VARCHAR NULL;\n"; + + @Override + public String getSQL() { + return PROJECTS_ADD_SWAP_COMPRESSION; + } +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapTime.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapTime.java new file mode 100644 index 0000000000..da89be8c25 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapTime.java @@ -0,0 +1,15 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; + +public class ProjectsAddSwapTime implements SQLUpdate { + private static final String PROJECTS_ADD_SWAP_TIME = + "ALTER TABLE `projects`\n" + + "ADD COLUMN `swap_time` DATETIME NULL;\n"; + + @Override + public String getSQL() { + return PROJECTS_ADD_SWAP_TIME; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java index c5f8d3f6e7..15042445ad 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java @@ -12,6 +12,9 @@ public class CreateProjectsTableSQLUpdate implements SQLUpdate { " `name` VARCHAR NOT NULL DEFAULT '',\n" + " `version_id` INT NOT NULL DEFAULT 0,\n" + " `last_accessed` DATETIME NULL DEFAULT 0,\n" + + " `swap_time` DATETIME NULL,\n" + + " `restore_time` DATETIME NULL,\n" + + " `swap_compression` VARCHAR NULL,\n" + " PRIMARY KEY (`name`)\n" + ")"; diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateRestore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateRestore.java new file mode 100644 index 0000000000..91abc5f473 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateRestore.java @@ -0,0 +1,40 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.LocalDateTime; + +public class UpdateRestore implements SQLUpdate { + private static final String UPDATE_RESTORE = + "UPDATE `projects`\n" + + "SET `last_accessed` = ?,\n" + + " `swap_time` = NULL,\n" + + " `restore_time` = ?,\n" + + " `swap_compression` = NULL\n" + + "WHERE `name` = ?;\n"; + + private final String projectName; + private final Timestamp now; + + public UpdateRestore(String projectName) { + this.projectName = projectName; + this.now = Timestamp.valueOf(LocalDateTime.now()); + } + + @Override + public String getSQL() { + return UPDATE_RESTORE; + } + + @Override + public void addParametersToStatement( + PreparedStatement statement + ) throws SQLException { + statement.setTimestamp(1, now); + statement.setTimestamp(2, now); + statement.setString(3, projectName); + } +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateSwap.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateSwap.java new file mode 100644 index 0000000000..24d7815279 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateSwap.java @@ -0,0 +1,42 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.LocalDateTime; + +public class UpdateSwap implements SQLUpdate { + private static final String UPDATE_SWAP = + "UPDATE `projects`\n" + + "SET `last_accessed` = NULL,\n" + + " `swap_time` = ?,\n" + + " `restore_time` = NULL,\n" + + " `swap_compression` = ?\n" + + "WHERE `name` = ?;\n"; + + private final String projectName; + private final String compression; + private final Timestamp now; + + public UpdateSwap(String projectName, String compression) { + this.projectName = projectName; + this.compression = compression; + this.now = Timestamp.valueOf(LocalDateTime.now()); + } + + @Override + public String getSQL() { + return UPDATE_SWAP; + } + + @Override + public void addParametersToStatement( + PreparedStatement statement + ) throws SQLException { + statement.setTimestamp(1, now); + statement.setString(2, compression); + statement.setString(3, projectName); + } +} From c67014b8591588cad53ade23899cb84fcfe42655 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jan 2021 14:00:48 +0000 Subject: [PATCH 09/12] Update SqliteDBStoreTest with swap/compression --- .../bridge/db/sqlite/SqliteDBStoreTest.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java index 02065c7b0c..9c88db88d8 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java @@ -69,10 +69,23 @@ public class SqliteDBStoreTest { assertEquals("asdf", dbStore.getOldestUnswappedProject()); } + @Test + public void swapAndRestore() { + String projectName = "something"; + String compression = "bzip2"; + dbStore.setLatestVersionForProject(projectName, 42); + dbStore.swap(projectName, compression); + assertNull(dbStore.getOldestUnswappedProject()); + assertEquals(dbStore.getSwapCompression(projectName), compression); + // and restore + dbStore.restore(projectName); + assertEquals(dbStore.getSwapCompression(projectName), null); + } + @Test public void noOldestProjectIfAllEvicted() { dbStore.setLatestVersionForProject("older", 3); - dbStore.setLastAccessedTime("older", null); + dbStore.swap("older", "bzip2"); assertNull(dbStore.getOldestUnswappedProject()); } @@ -93,7 +106,7 @@ public class SqliteDBStoreTest { ) ); assertEquals("older", dbStore.getOldestUnswappedProject()); - dbStore.setLastAccessedTime("older", null); + dbStore.swap("older", "bzip2"); assertEquals("newer", dbStore.getOldestUnswappedProject()); } @@ -115,9 +128,9 @@ public class SqliteDBStoreTest { Timestamp.valueOf(LocalDateTime.now()) ); assertEquals(1, dbStore.getNumUnswappedProjects()); - dbStore.setLastAccessedTime( + dbStore.swap( "asdf", - null + "bzip2" ); assertEquals(0, dbStore.getNumUnswappedProjects()); } @@ -143,8 +156,8 @@ public class SqliteDBStoreTest { @Test public void projectStateIsSwappedIfLastAccessedIsNull() { dbStore.setLatestVersionForProject("asdf", 1); - dbStore.setLastAccessedTime("asdf", null); + dbStore.swap("asdf", "bzip2"); assertEquals(ProjectState.SWAPPED, dbStore.getProjectState("asdf")); } -} \ No newline at end of file +} From b92c737814ac9499c004eecacdf3b34c87d7a527 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jan 2021 14:02:22 +0000 Subject: [PATCH 10/12] Use configured compression method in swap job --- .../bridge/swap/job/SwapJobImpl.java | 46 +++++++++++++++---- .../WLGitBridgeIntegrationTest.java | 2 +- .../bridge/swap/job/SwapJobImplTest.java | 34 +++++++++++++- 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java index f989fe1341..e6524de010 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java @@ -34,6 +34,7 @@ public class SwapJobImpl implements SwapJob { private final RepoStore repoStore; private final DBStore dbStore; private final SwapStore swapStore; + private final CompressionMethod compressionMethod; private final Timer timer; @@ -51,6 +52,7 @@ public class SwapJobImpl implements SwapJob { GiB * cfg.getLowGiB(), GiB * cfg.getHighGiB(), Duration.ofMillis(cfg.getIntervalMillis()), + cfg.getCompressionMethod(), lock, repoStore, dbStore, @@ -63,6 +65,7 @@ public class SwapJobImpl implements SwapJob { long lowWatermarkBytes, long highWatermarkBytes, Duration interval, + CompressionMethod method, ProjectLock lock, RepoStore repoStore, DBStore dbStore, @@ -72,6 +75,7 @@ public class SwapJobImpl implements SwapJob { this.lowWatermarkBytes = lowWatermarkBytes; this.highWatermarkBytes = highWatermarkBytes; this.interval = interval; + this.compressionMethod = method; this.lock = lock; this.repoStore = repoStore; this.dbStore = dbStore; @@ -196,15 +200,29 @@ public class SwapJobImpl implements SwapJob { Log.error("[{}] Exception while running gc on project: {}", projName, e); } long[] sizePtr = new long[1]; - try (InputStream blob = repoStore.bzip2Project(projName, sizePtr)) { + try (InputStream blob = getBlobStream(projName, sizePtr)) { swapStore.upload(projName, blob, sizePtr[0]); - dbStore.setLastAccessedTime(projName, null); + String compression = SwapJob.compressionMethodAsString(compressionMethod); + if (compression == null) { + throw new RuntimeException("invalid compression method, should not happen"); + } + dbStore.swap(projName, compression); repoStore.remove(projName); } } Log.info("Evicted project: {}", projName); } + private InputStream getBlobStream(String projName, long[] sizePtr) throws IOException { + if (compressionMethod == CompressionMethod.Gzip) { + return repoStore.gzipProject(projName, sizePtr); + } else if (compressionMethod == CompressionMethod.Bzip2) { + return repoStore.bzip2Project(projName, sizePtr); + } else { + throw new RuntimeException("invalid compression method, should not happen"); + } + } + /** * @see SwapJob#restore(String) for high-level description. * @@ -220,15 +238,23 @@ public class SwapJobImpl implements SwapJob { public void restore(String projName) throws IOException { try (LockGuard __ = lock.lockGuard(projName)) { try (InputStream zipped = swapStore.openDownloadStream(projName)) { - repoStore.unbzip2Project( - projName, - zipped - ); + String compression = dbStore.getSwapCompression(projName); + if (compression == null) { + throw new RuntimeException("Missing compression method during restore, should not happen"); + } + if ("gzip".equals(compression)) { + repoStore.ungzipProject( + projName, + zipped + ); + } else if ("bzip2".equals(compression)) { + repoStore.unbzip2Project( + projName, + zipped + ); + } swapStore.remove(projName); - dbStore.setLastAccessedTime( - projName, - Timestamp.valueOf(LocalDateTime.now()) - ); + dbStore.restore(projName); } } } diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java index 5ee773bede..31eda92198 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java @@ -667,7 +667,7 @@ public class WLGitBridgeIntegrationTest { server.start(); server.setState(states.get("wlgbCanSwapProjects").get("state")); wlgb = new GitBridgeApp(new String[] { - makeConfigFile(33874, 3874, new SwapJobConfig(1, 0, 0, 250)) + makeConfigFile(33874, 3874, new SwapJobConfig(1, 0, 0, 250, null)) }); wlgb.run(); File rootGitDir = new File(wlgb.config.getRootGitDirectory()); diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java index a9154d47a0..a3a26c3400 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java @@ -68,6 +68,7 @@ public class SwapJobImplTest { 15000, 30000, Duration.ofMillis(100), + SwapJob.CompressionMethod.Bzip2, lock, repoStore, dbStore, @@ -120,11 +121,42 @@ public class SwapJobImplTest { while (swapJob.swaps.get() < 1); assertEquals(1, dbStore.getNumUnswappedProjects()); assertEquals("proj1", dbStore.getOldestUnswappedProject()); + assertEquals("bzip2", dbStore.getSwapCompression("proj2")); swapJob.restore("proj2"); + assertEquals(null, dbStore.getSwapCompression("proj2")); int numSwaps = swapJob.swaps.get(); while (swapJob.swaps.get() <= numSwaps); assertEquals(1, dbStore.getNumUnswappedProjects()); assertEquals("proj2", dbStore.getOldestUnswappedProject()); } -} \ No newline at end of file + @Test + public void swapCompressionGzip() throws IOException { + swapJob = new SwapJobImpl( + 1, + 15000, + 30000, + Duration.ofMillis(100), + SwapJob.CompressionMethod.Gzip, + lock, + repoStore, + dbStore, + swapStore + ); + swapJob.lowWatermarkBytes = 16384; + assertEquals(2, dbStore.getNumUnswappedProjects()); + assertEquals("proj2", dbStore.getOldestUnswappedProject()); + swapJob.start(); + while (swapJob.swaps.get() < 1); + assertEquals(1, dbStore.getNumUnswappedProjects()); + assertEquals("proj1", dbStore.getOldestUnswappedProject()); + assertEquals("gzip", dbStore.getSwapCompression("proj2")); + swapJob.restore("proj2"); + assertEquals(null, dbStore.getSwapCompression("proj2")); + int numSwaps = swapJob.swaps.get(); + while (swapJob.swaps.get() <= numSwaps); + assertEquals(1, dbStore.getNumUnswappedProjects()); + assertEquals("proj2", dbStore.getOldestUnswappedProject()); + } + +} From b0106224425c7152b9e00d407d888630783e7be7 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 21 Jan 2021 11:23:50 +0000 Subject: [PATCH 11/12] Change default compressionMethod to gzip --- services/git-bridge/conf/example_config.json | 2 +- services/git-bridge/conf/local.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/git-bridge/conf/example_config.json b/services/git-bridge/conf/example_config.json index c716e998b4..715346e57b 100644 --- a/services/git-bridge/conf/example_config.json +++ b/services/git-bridge/conf/example_config.json @@ -27,6 +27,6 @@ "lowGiB": 128, "highGiB": 256, "intervalMillis": 3600000, - "compressionMethod": "bzip2" + "compressionMethod": "gzip" } } diff --git a/services/git-bridge/conf/local.json b/services/git-bridge/conf/local.json index 17cc37796f..e4f2fdafa9 100644 --- a/services/git-bridge/conf/local.json +++ b/services/git-bridge/conf/local.json @@ -23,6 +23,6 @@ "lowGiB": 128, "highGiB": 256, "intervalMillis": 3600000, - "compressionMethod": "bzip2" + "compressionMethod": "gzip" } } From 465a39c564276d22d02bc3cd171fe8a16c805167 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 22 Jan 2021 15:32:34 +0000 Subject: [PATCH 12/12] Wait one second between checks in swap test --- .../bridge/swap/job/SwapJobImplTest.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java index a3a26c3400..3e9aa2e491 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java @@ -83,13 +83,17 @@ public class SwapJobImplTest { } } + private void waitASecond() { + try { Thread.sleep(1 * 1000); } catch (Exception _e) {} + } + @Test public void startingTimerAlwaysCausesASwap() { swapJob.lowWatermarkBytes = 16384; swapJob.interval = Duration.ofHours(1); assertEquals(0, swapJob.swaps.get()); swapJob.start(); - while (swapJob.swaps.get() <= 0); + do { waitASecond(); } while (swapJob.swaps.get() <= 0); assertTrue(swapJob.swaps.get() > 0); } @@ -98,7 +102,7 @@ public class SwapJobImplTest { swapJob.lowWatermarkBytes = 16384; assertEquals(0, swapJob.swaps.get()); swapJob.start(); - while (swapJob.swaps.get() <= 1); + do { waitASecond(); } while (swapJob.swaps.get() <= 1); assertTrue(swapJob.swaps.get() > 1); } @@ -107,7 +111,7 @@ public class SwapJobImplTest { swapJob.highWatermarkBytes = 65536; assertEquals(2, dbStore.getNumUnswappedProjects()); swapJob.start(); - while (swapJob.swaps.get() < 1); + do { waitASecond(); } while (swapJob.swaps.get() < 1); assertEquals(2, dbStore.getNumUnswappedProjects()); } @@ -118,14 +122,14 @@ public class SwapJobImplTest { assertEquals(2, dbStore.getNumUnswappedProjects()); assertEquals("proj2", dbStore.getOldestUnswappedProject()); swapJob.start(); - while (swapJob.swaps.get() < 1); + do { waitASecond(); } while (swapJob.swaps.get() < 1); assertEquals(1, dbStore.getNumUnswappedProjects()); assertEquals("proj1", dbStore.getOldestUnswappedProject()); assertEquals("bzip2", dbStore.getSwapCompression("proj2")); swapJob.restore("proj2"); assertEquals(null, dbStore.getSwapCompression("proj2")); int numSwaps = swapJob.swaps.get(); - while (swapJob.swaps.get() <= numSwaps); + do { waitASecond(); } while (swapJob.swaps.get() <= numSwaps); assertEquals(1, dbStore.getNumUnswappedProjects()); assertEquals("proj2", dbStore.getOldestUnswappedProject()); } @@ -147,14 +151,14 @@ public class SwapJobImplTest { assertEquals(2, dbStore.getNumUnswappedProjects()); assertEquals("proj2", dbStore.getOldestUnswappedProject()); swapJob.start(); - while (swapJob.swaps.get() < 1); + do { waitASecond(); } while (swapJob.swaps.get() < 1); assertEquals(1, dbStore.getNumUnswappedProjects()); assertEquals("proj1", dbStore.getOldestUnswappedProject()); assertEquals("gzip", dbStore.getSwapCompression("proj2")); swapJob.restore("proj2"); assertEquals(null, dbStore.getSwapCompression("proj2")); int numSwaps = swapJob.swaps.get(); - while (swapJob.swaps.get() <= numSwaps); + do { waitASecond(); } while (swapJob.swaps.get() <= numSwaps); assertEquals(1, dbStore.getNumUnswappedProjects()); assertEquals("proj2", dbStore.getOldestUnswappedProject()); }