github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-03-03 05:22:44 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #91 from overleaf/sk-gzip

Browse source 
Gzip Compression in Swap Job
This commit is contained in:
Shane Kilkelly 2021-02-04 13:44:32 +00:00 committed by GitHub
commit a079ba2c6e
24 changed files with 548 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
services/git-bridge/conf/example_config.json
View file

@ -26,6 +26,7 @@
"minProjects": 50,
"lowGiB": 128,
"highGiB": 256,
"intervalMillis": 3600000
"intervalMillis": 3600000,
"compressionMethod": "gzip"
}
}

3
services/git-bridge/conf/local.json
View file

@ -22,6 +22,7 @@
"minProjects": 50,
"lowGiB": 128,
"highGiB": 256,
"intervalMillis": 3600000
"intervalMillis": 3600000,
"compressionMethod": "gzip"
}
}

6
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java
View file

@ -24,6 +24,12 @@ public interface DBStore {
String getOldestUnswappedProject();
void swap(String projectName, String compressionMethod);
void restore(String projectName);
String getSwapCompression(String projectName);
int getNumUnswappedProjects();
ProjectState getProjectState(String projectName);

11
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/noop/NoopDbStore.java
View file

@ -60,7 +60,16 @@ public class NoopDbStore implements DBStore {
@Override
public void setLastAccessedTime(String projectName, Timestamp time) {
}
@Override
public void swap(String projectName, String compressionMethod) {}
@Override
public void restore(String projectName) {}
@Override
public String getSwapCompression(String projectName) {
return null;
}
}

34
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java
View file

@ -104,6 +104,21 @@ public class SqliteDBStore implements DBStore {
update(new SetProjectLastAccessedTime(projectName, lastAccessed));
}
@Override
public void swap(String projectName, String compressionMethod) {
update(new UpdateSwap(projectName, compressionMethod));
}
@Override
public void restore(String projectName) {
update(new UpdateRestore(projectName));
}
@Override
public String getSwapCompression(String projectName) {
return query(new GetSwapCompression(projectName));
}
private Connection openConnectionTo(File dbFile) {
File parentDir = dbFile.getParentFile();
if (!parentDir.exists() && !parentDir.mkdirs()) {
@ -127,20 +142,27 @@ public class SqliteDBStore implements DBStore {
}
private void createTables() {
try {
doUpdate(new ProjectsAddLastAccessed());
} catch (SQLException ignore) {
/* We need to eat exceptions from here */
}
/* Migrations */
/* We need to eat exceptions from here */
try { doUpdate(new ProjectsAddLastAccessed()); } catch (SQLException ignore) {}
try { doUpdate(new ProjectsAddSwapTime()); } catch (SQLException ignore) {}
try { doUpdate(new ProjectsAddRestoreTime()); } catch (SQLException ignore) {}
try { doUpdate(new ProjectsAddSwapCompression()); } catch (SQLException ignore) {}
/* Create tables (if they don't exist) */
Stream.of(
new CreateProjectsTableSQLUpdate(),
new CreateProjectsIndexLastAccessed(),
new CreateURLIndexStoreSQLUpdate(),
new CreateIndexURLIndexStore()
).forEach(this::update);
/* In the case of needing to change the schema, we need to check that
ProjectsAddLastAccessed didn't just fail */
migrations didn't just fail */
Preconditions.checkState(query(new LastAccessedColumnExists()));
Preconditions.checkState(query(new SwapTimeColumnExists()));
Preconditions.checkState(query(new RestoreTimeColumnExists()));
Preconditions.checkState(query(new SwapCompressionColumnExists()));
}
private void update(SQLUpdate update) {

39
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetSwapCompression.java Normal file
View file

@ -0,0 +1,39 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
public class GetSwapCompression implements SQLQuery<String> {
private static final String GET_SWAP_COMPRESSION =
"SELECT `swap_compression` FROM `projects` WHERE `name` = ?";
private final String projectName;
public GetSwapCompression(String projectName) {
this.projectName = projectName;
}
@Override
public String processResultSet(ResultSet resultSet) throws SQLException {
String compression = null;
while (resultSet.next()) {
compression = resultSet.getString("swap_compression");
}
return compression;
}
@Override
public String getSQL() {
return GET_SWAP_COMPRESSION;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
}
}

26
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/RestoreTimeColumnExists.java Normal file
View file

@ -0,0 +1,26 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
public class RestoreTimeColumnExists implements SQLQuery<Boolean> {
private static final String RESTORE_TIME_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return RESTORE_TIME_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("restore_time")) {
return true;
}
}
return false;
}
}

27
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapCompressionColumnExists.java Normal file
View file

@ -0,0 +1,27 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
public class SwapCompressionColumnExists implements SQLQuery<Boolean> {
private static final String SWAP_COMPRESSION_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return SWAP_COMPRESSION_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("swap_compression")) {
return true;
}
}
return false;
}
}

27
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapTimeColumnExists.java Normal file
View file

@ -0,0 +1,27 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
public class SwapTimeColumnExists implements SQLQuery<Boolean> {
private static final String SWAP_TIME_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return SWAP_TIME_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("swap_time")) {
return true;
}
}
return false;
}
}

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddRestoreTime.java Normal file
View file

@ -0,0 +1,14 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class ProjectsAddRestoreTime implements SQLUpdate {
private static final String PROJECTS_ADD_RESTORE_TIME =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `restore_time` DATETIME NULL;\n";
@Override
public String getSQL() {
return PROJECTS_ADD_RESTORE_TIME;
}
}

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapCompression.java Normal file
View file

@ -0,0 +1,14 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class ProjectsAddSwapCompression implements SQLUpdate {
private static final String PROJECTS_ADD_SWAP_COMPRESSION =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `swap_compression` VARCHAR NULL;\n";
@Override
public String getSQL() {
return PROJECTS_ADD_SWAP_COMPRESSION;
}
}

15
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapTime.java Normal file
View file

@ -0,0 +1,15 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class ProjectsAddSwapTime implements SQLUpdate {
private static final String PROJECTS_ADD_SWAP_TIME =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `swap_time` DATETIME NULL;\n";
@Override
public String getSQL() {
return PROJECTS_ADD_SWAP_TIME;
}
}

3
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java
View file

@ -12,6 +12,9 @@ public class CreateProjectsTableSQLUpdate implements SQLUpdate {
" `name` VARCHAR NOT NULL DEFAULT '',\n" +
" `version_id` INT NOT NULL DEFAULT 0,\n" +
" `last_accessed` DATETIME NULL DEFAULT 0,\n" +
" `swap_time` DATETIME NULL,\n" +
" `restore_time` DATETIME NULL,\n" +
" `swap_compression` VARCHAR NULL,\n" +
" PRIMARY KEY (`name`)\n" +
")";

40
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateRestore.java Normal file
View file

@ -0,0 +1,40 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
public class UpdateRestore implements SQLUpdate {
private static final String UPDATE_RESTORE =
"UPDATE `projects`\n" +
"SET `last_accessed` = ?,\n" +
" `swap_time` = NULL,\n" +
" `restore_time` = ?,\n" +
" `swap_compression` = NULL\n" +
"WHERE `name` = ?;\n";
private final String projectName;
private final Timestamp now;
public UpdateRestore(String projectName) {
this.projectName = projectName;
this.now = Timestamp.valueOf(LocalDateTime.now());
}
@Override
public String getSQL() {
return UPDATE_RESTORE;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setTimestamp(1, now);
statement.setTimestamp(2, now);
statement.setString(3, projectName);
}
}

42
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateSwap.java Normal file
View file

@ -0,0 +1,42 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
public class UpdateSwap implements SQLUpdate {
private static final String UPDATE_SWAP =
"UPDATE `projects`\n" +
"SET `last_accessed` = NULL,\n" +
" `swap_time` = ?,\n" +
" `restore_time` = NULL,\n" +
" `swap_compression` = ?\n" +
"WHERE `name` = ?;\n";
private final String projectName;
private final String compression;
private final Timestamp now;
public UpdateSwap(String projectName, String compression) {
this.projectName = projectName;
this.compression = compression;
this.now = Timestamp.valueOf(LocalDateTime.now());
}
@Override
public String getSQL() {
return UPDATE_SWAP;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setTimestamp(1, now);
statement.setString(2, compression);
statement.setString(3, projectName);
}
}

32
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java
View file

@ -136,9 +136,20 @@ public class FSGitRepoStore implements RepoStore {
long[] sizePtr
) throws IOException {
Project.checkValidProjectName(projectName);
Log.info("[{}] bzip2 project", projectName);
return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr);
}
@Override
public InputStream gzipProject(
String projectName,
long[] sizePtr
) throws IOException {
Project.checkValidProjectName(projectName);
Log.info("[{}] gzip project", projectName);
return Tar.gzip.zip(getDotGitForProject(projectName), sizePtr);
}
@Override
public void gcProject(String projectName) throws IOException {
Project.checkValidProjectName(projectName);
@ -168,9 +179,30 @@ public class FSGitRepoStore implements RepoStore {
"evicted project already exist",
projectName
);
Log.info("[{}] un-bzip2 project", projectName);
Tar.bz2.unzip(dataStream, getDirForProject(projectName));
}
@Override
public void ungzipProject(
String projectName,
InputStream dataStream
) throws IOException {
Preconditions.checkArgument(
Project.isValidProjectName(projectName),
"[%s] invalid project name: ",
projectName
);
Preconditions.checkState(
getDirForProject(projectName).mkdirs(),
"[%s] directories for " +
"evicted project already exist",
projectName
);
Log.info("[{}] un-gzip project", projectName);
Tar.gzip.unzip(dataStream, getDirForProject(projectName));
}
private File getDirForProject(String projectName) {
Project.checkValidProjectName(projectName);
return Paths.get(

28
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java
View file

@ -50,6 +50,22 @@ public interface RepoStore {
return bzip2Project(projectName, null);
}
/**
* Tars and gzips the .git directory of the given project. Throws an
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().
*/
InputStream gzipProject(
String projectName,
long[] sizePtr
) throws IOException;
default InputStream gzipProject(
String projectName
) throws IOException {
return gzipProject(projectName, null);
}
void gcProject(String projectName) throws IOException;
/**
@ -73,4 +89,16 @@ public interface RepoStore {
InputStream dataStream
) throws IOException;
/**
* Ungzips the given data stream into a .git directory for projectName.
* Creates the project's git directory.
* If projectName already exists, throws an IOException.
* @param projectName the name of the project, e.g. abc123
* @param dataStream the data stream containing the gzip contents.
*/
void ungzipProject(
String projectName,
InputStream dataStream
) throws IOException;
}

40
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java
View file

@ -13,6 +13,46 @@ import java.util.Optional;
*/
public interface SwapJob {
enum CompressionMethod { Bzip2, Gzip }
static CompressionMethod stringToCompressionMethod(String compressionString) {
if (compressionString == null) {
return null;
}
CompressionMethod result;
switch (compressionString) {
case "gzip":
result = CompressionMethod.Gzip;
break;
case "bzip2":
result = CompressionMethod.Bzip2;
break;
default:
result = null;
break;
}
return result;
}
static String compressionMethodAsString(CompressionMethod compressionMethod) {
if (compressionMethod == null) {
return null;
}
String result;
switch (compressionMethod) {
case Gzip:
result = "gzip";
break;
case Bzip2:
result = "bzip2";
break;
default:
result = null;
break;
}
return result;
}
static SwapJob fromConfig(
Optional<SwapJobConfig> cfg,
ProjectLock lock,

16
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java
View file

@ -1,5 +1,8 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob.CompressionMethod;
/**
* Created by winston on 23/08/2016.
*/
@ -9,17 +12,20 @@ public class SwapJobConfig {
private final int lowGiB;
private final int highGiB;
private final long intervalMillis;
private final String compressionMethod;
public SwapJobConfig(
int minProjects,
int lowGiB,
int highGiB,
long intervalMillis
long intervalMillis,
String compressionMethod
) {
this.minProjects = minProjects;
this.lowGiB = lowGiB;
this.highGiB = highGiB;
this.intervalMillis = intervalMillis;
this.compressionMethod = compressionMethod;
}
public int getMinProjects() {
@ -38,4 +44,12 @@ public class SwapJobConfig {
return intervalMillis;
}
public SwapJob.CompressionMethod getCompressionMethod() {
CompressionMethod result = SwapJob.stringToCompressionMethod(compressionMethod);
if (result == null) {
Log.info("SwapJobConfig: un-supported compressionMethod '{}', default to 'bzip2'", compressionMethod);
result = CompressionMethod.Bzip2;
}
return result;
}
}

46
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java
View file

@ -34,6 +34,7 @@ public class SwapJobImpl implements SwapJob {
private final RepoStore repoStore;
private final DBStore dbStore;
private final SwapStore swapStore;
private final CompressionMethod compressionMethod;
private final Timer timer;
@ -51,6 +52,7 @@ public class SwapJobImpl implements SwapJob {
GiB * cfg.getLowGiB(),
GiB * cfg.getHighGiB(),
Duration.ofMillis(cfg.getIntervalMillis()),
cfg.getCompressionMethod(),
lock,
repoStore,
dbStore,
@ -63,6 +65,7 @@ public class SwapJobImpl implements SwapJob {
long lowWatermarkBytes,
long highWatermarkBytes,
Duration interval,
CompressionMethod method,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
@ -72,6 +75,7 @@ public class SwapJobImpl implements SwapJob {
this.lowWatermarkBytes = lowWatermarkBytes;
this.highWatermarkBytes = highWatermarkBytes;
this.interval = interval;
this.compressionMethod = method;
this.lock = lock;
this.repoStore = repoStore;
this.dbStore = dbStore;
@ -196,15 +200,29 @@ public class SwapJobImpl implements SwapJob {
Log.error("[{}] Exception while running gc on project: {}", projName, e);
}
long[] sizePtr = new long[1];
try (InputStream blob = repoStore.bzip2Project(projName, sizePtr)) {
try (InputStream blob = getBlobStream(projName, sizePtr)) {
swapStore.upload(projName, blob, sizePtr[0]);
dbStore.setLastAccessedTime(projName, null);
String compression = SwapJob.compressionMethodAsString(compressionMethod);
if (compression == null) {
throw new RuntimeException("invalid compression method, should not happen");
}
dbStore.swap(projName, compression);
repoStore.remove(projName);
}
}
Log.info("Evicted project: {}", projName);
}
private InputStream getBlobStream(String projName, long[] sizePtr) throws IOException {
if (compressionMethod == CompressionMethod.Gzip) {
return repoStore.gzipProject(projName, sizePtr);
} else if (compressionMethod == CompressionMethod.Bzip2) {
return repoStore.bzip2Project(projName, sizePtr);
} else {
throw new RuntimeException("invalid compression method, should not happen");
}
}
/**
* @see SwapJob#restore(String) for high-level description.
*
@ -220,15 +238,23 @@ public class SwapJobImpl implements SwapJob {
public void restore(String projName) throws IOException {
try (LockGuard __ = lock.lockGuard(projName)) {
try (InputStream zipped = swapStore.openDownloadStream(projName)) {
repoStore.unbzip2Project(
projName,
zipped
);
String compression = dbStore.getSwapCompression(projName);
if (compression == null) {
throw new RuntimeException("Missing compression method during restore, should not happen");
}
if ("gzip".equals(compression)) {
repoStore.ungzipProject(
projName,
zipped
);
} else if ("bzip2".equals(compression)) {
repoStore.unbzip2Project(
projName,
zipped
);
}
swapStore.remove(projName);
dbStore.setLastAccessedTime(
projName,
Timestamp.valueOf(LocalDateTime.now())
);
dbStore.restore(projName);
}
}
}

40
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java
View file

@ -6,6 +6,8 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.io.FileUtils;
@ -23,6 +25,44 @@ import java.nio.file.Paths;
* Caller is responsible for all resources.
*/
public class Tar {
public static class gzip {
public static InputStream zip(
File fileOrDir
) throws IOException {
return zip(fileOrDir, null);
}
public static InputStream zip(
File fileOrDir,
long[] sizePtr
) throws IOException {
File tmp = File.createTempFile(fileOrDir.getName(), ".tar.gz");
tmp.deleteOnExit();
OutputStream target = new FileOutputStream(tmp);
/* Closes target */
try (OutputStream gz = new GzipCompressorOutputStream(target)) {
tarTo(fileOrDir, gz);
} catch (IOException e) {
tmp.delete();
throw e;
}
if (sizePtr != null) {
sizePtr[0] = tmp.length();
}
return new DeletingFileInputStream(tmp);
}
public static void unzip(
InputStream targz,
File parentDir
) throws IOException {
/* GzipCompressorInputStream does not need closing
Closing it would close targz which we should not do */
InputStream tar = new GzipCompressorInputStream(targz);
untar(tar, parentDir);
}
}
public static class bz2 {

2
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java
View file

@ -667,7 +667,7 @@ public class WLGitBridgeIntegrationTest {
server.start();
server.setState(states.get("wlgbCanSwapProjects").get("state"));
wlgb = new GitBridgeApp(new String[] {
makeConfigFile(33874, 3874, new SwapJobConfig(1, 0, 0, 250))
makeConfigFile(33874, 3874, new SwapJobConfig(1, 0, 0, 250, null))
});
wlgb.run();
File rootGitDir = new File(wlgb.config.getRootGitDirectory());

25
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java
View file

@ -69,10 +69,23 @@ public class SqliteDBStoreTest {
assertEquals("asdf", dbStore.getOldestUnswappedProject());
}
@Test
public void swapAndRestore() {
String projectName = "something";
String compression = "bzip2";
dbStore.setLatestVersionForProject(projectName, 42);
dbStore.swap(projectName, compression);
assertNull(dbStore.getOldestUnswappedProject());
assertEquals(dbStore.getSwapCompression(projectName), compression);
// and restore
dbStore.restore(projectName);
assertEquals(dbStore.getSwapCompression(projectName), null);
}
@Test
public void noOldestProjectIfAllEvicted() {
dbStore.setLatestVersionForProject("older", 3);
dbStore.setLastAccessedTime("older", null);
dbStore.swap("older", "bzip2");
assertNull(dbStore.getOldestUnswappedProject());
}
@ -93,7 +106,7 @@ public class SqliteDBStoreTest {
)
);
assertEquals("older", dbStore.getOldestUnswappedProject());
dbStore.setLastAccessedTime("older", null);
dbStore.swap("older", "bzip2");
assertEquals("newer", dbStore.getOldestUnswappedProject());
}
@ -115,9 +128,9 @@ public class SqliteDBStoreTest {
Timestamp.valueOf(LocalDateTime.now())
);
assertEquals(1, dbStore.getNumUnswappedProjects());
dbStore.setLastAccessedTime(
dbStore.swap(
"asdf",
null
"bzip2"
);
assertEquals(0, dbStore.getNumUnswappedProjects());
}
@ -143,8 +156,8 @@ public class SqliteDBStoreTest {
@Test
public void projectStateIsSwappedIfLastAccessedIsNull() {
dbStore.setLatestVersionForProject("asdf", 1);
dbStore.setLastAccessedTime("asdf", null);
dbStore.swap("asdf", "bzip2");
assertEquals(ProjectState.SWAPPED, dbStore.getProjectState("asdf"));
}
}
}

48
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java
View file

@ -68,6 +68,7 @@ public class SwapJobImplTest {
15000,
30000,
Duration.ofMillis(100),
SwapJob.CompressionMethod.Bzip2,
lock,
repoStore,
dbStore,
@ -82,13 +83,17 @@ public class SwapJobImplTest {
}
}
private void waitASecond() {
try { Thread.sleep(1 * 1000); } catch (Exception _e) {}
}
@Test
public void startingTimerAlwaysCausesASwap() {
swapJob.lowWatermarkBytes = 16384;
swapJob.interval = Duration.ofHours(1);
assertEquals(0, swapJob.swaps.get());
swapJob.start();
while (swapJob.swaps.get() <= 0);
do { waitASecond(); } while (swapJob.swaps.get() <= 0);
assertTrue(swapJob.swaps.get() > 0);
}
@ -97,7 +102,7 @@ public class SwapJobImplTest {
swapJob.lowWatermarkBytes = 16384;
assertEquals(0, swapJob.swaps.get());
swapJob.start();
while (swapJob.swaps.get() <= 1);
do { waitASecond(); } while (swapJob.swaps.get() <= 1);
assertTrue(swapJob.swaps.get() > 1);
}
@ -106,7 +111,7 @@ public class SwapJobImplTest {
swapJob.highWatermarkBytes = 65536;
assertEquals(2, dbStore.getNumUnswappedProjects());
swapJob.start();
while (swapJob.swaps.get() < 1);
do { waitASecond(); } while (swapJob.swaps.get() < 1);
assertEquals(2, dbStore.getNumUnswappedProjects());
}
@ -117,14 +122,45 @@ public class SwapJobImplTest {
assertEquals(2, dbStore.getNumUnswappedProjects());
assertEquals("proj2", dbStore.getOldestUnswappedProject());
swapJob.start();
while (swapJob.swaps.get() < 1);
do { waitASecond(); } while (swapJob.swaps.get() < 1);
assertEquals(1, dbStore.getNumUnswappedProjects());
assertEquals("proj1", dbStore.getOldestUnswappedProject());
assertEquals("bzip2", dbStore.getSwapCompression("proj2"));
swapJob.restore("proj2");
assertEquals(null, dbStore.getSwapCompression("proj2"));
int numSwaps = swapJob.swaps.get();
while (swapJob.swaps.get() <= numSwaps);
do { waitASecond(); } while (swapJob.swaps.get() <= numSwaps);
assertEquals(1, dbStore.getNumUnswappedProjects());
assertEquals("proj2", dbStore.getOldestUnswappedProject());
}
}
@Test
public void swapCompressionGzip() throws IOException {
swapJob = new SwapJobImpl(
1,
15000,
30000,
Duration.ofMillis(100),
SwapJob.CompressionMethod.Gzip,
lock,
repoStore,
dbStore,
swapStore
);
swapJob.lowWatermarkBytes = 16384;
assertEquals(2, dbStore.getNumUnswappedProjects());
assertEquals("proj2", dbStore.getOldestUnswappedProject());
swapJob.start();
do { waitASecond(); } while (swapJob.swaps.get() < 1);
assertEquals(1, dbStore.getNumUnswappedProjects());
assertEquals("proj1", dbStore.getOldestUnswappedProject());
assertEquals("gzip", dbStore.getSwapCompression("proj2"));
swapJob.restore("proj2");
assertEquals(null, dbStore.getSwapCompression("proj2"));
int numSwaps = swapJob.swaps.get();
do { waitASecond(); } while (swapJob.swaps.get() <= numSwaps);
assertEquals(1, dbStore.getNumUnswappedProjects());
assertEquals("proj2", dbStore.getOldestUnswappedProject());
}
}