diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java index d9b0a57f58..a3646c1b7b 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java @@ -10,6 +10,7 @@ import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache; import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotAPI; import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI; import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob; +import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobConfig; import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobImpl; import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore; import uk.ac.ic.wlgitbridge.data.CandidateSnapshot; @@ -43,18 +44,19 @@ public class Bridge { private final RepoStore repoStore; private final DBStore dbStore; private final SwapStore swapStore; + private final SwapJob swapJob; private final SnapshotAPI snapshotAPI; private final ResourceCache resourceCache; - private final SwapJob swapJob; private final PostbackManager postbackManager; public static Bridge make( RepoStore repoStore, DBStore dbStore, - SwapStore swapStore + SwapStore swapStore, + SwapJobConfig swapJobConfig ) { ProjectLock lock = new ProjectLockImpl((int threads) -> Log.info("Waiting for " + threads + " projects...") @@ -64,14 +66,15 @@ public class Bridge { repoStore, dbStore, swapStore, - new NetSnapshotAPI(), - new UrlResourceCache(dbStore), new SwapJobImpl( + swapJobConfig, lock, repoStore, dbStore, swapStore - ) + ), + new NetSnapshotAPI(), + new UrlResourceCache(dbStore) ); } @@ -80,9 +83,9 @@ public class Bridge { RepoStore repoStore, DBStore dbStore, SwapStore swapStore, + SwapJob swapJob, SnapshotAPI snapshotAPI, - ResourceCache resourceCache, - SwapJob swapJob + ResourceCache resourceCache ) { this.lock = lock; this.repoStore = repoStore; diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java index c59a8fc195..79267596c8 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java @@ -8,6 +8,8 @@ import java.util.List; */ public interface DBStore { + int getNumProjects(); + List getProjectNames(); void setLatestVersionForProject(String project, int versionID); diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java index d69c22b51e..1b4566b54d 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java @@ -2,10 +2,7 @@ package uk.ac.ic.wlgitbridge.bridge.db.sqlite; import uk.ac.ic.wlgitbridge.bridge.db.DBInitException; import uk.ac.ic.wlgitbridge.bridge.db.DBStore; -import uk.ac.ic.wlgitbridge.bridge.db.sqlite.query.GetLatestVersionForProjectSQLQuery; -import uk.ac.ic.wlgitbridge.bridge.db.sqlite.query.GetOldestProjectName; -import uk.ac.ic.wlgitbridge.bridge.db.sqlite.query.GetPathForURLInProjectSQLQuery; -import uk.ac.ic.wlgitbridge.bridge.db.sqlite.query.GetProjectNamesSQLQuery; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.query.*; import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.*; import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete.DeleteFilesForProjectSQLUpdate; import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert.AddURLIndexSQLUpdate; @@ -34,6 +31,11 @@ public class SqliteDBStore implements DBStore { } } + @Override + public int getNumProjects() { + return query(new GetNumProjects()); + } + @Override public List getProjectNames() { return query(new GetProjectNamesSQLQuery()); diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumProjects.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumProjects.java new file mode 100644 index 0000000000..e5123c2ea1 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumProjects.java @@ -0,0 +1,30 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by winston on 24/08/2016. + */ +public class GetNumProjects implements SQLQuery { + + private static final String GET_NUM_PROJECTS = + "SELECT COUNT(*)\n" + + " FROM `projects`"; + + @Override + public String getSQL() { + return GET_NUM_PROJECTS; + } + + @Override + public Integer processResultSet(ResultSet resultSet) throws SQLException { + while (resultSet.next()) { + return resultSet.getInt("COUNT(*)"); + } + throw new IllegalStateException("Count always returns results"); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java index 6c89a2915a..ab5b432346 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java @@ -9,8 +9,8 @@ public class CreateProjectsTableSQLUpdate implements SQLUpdate { private static final String CREATE_PROJECTS_TABLE = "CREATE TABLE IF NOT EXISTS `projects` (\n" + - " `name` varchar(10) NOT NULL DEFAULT '',\n" + - " `version_id` int(11) NOT NULL DEFAULT 0,\n" + + " `name` VARCHAR NOT NULL DEFAULT '',\n" + + " `version_id` INT NOT NULL DEFAULT 0,\n" + " PRIMARY KEY (`name`)\n" + ")"; @Override diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobConfig.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobConfig.java new file mode 100644 index 0000000000..5a6d37302a --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobConfig.java @@ -0,0 +1,33 @@ +package uk.ac.ic.wlgitbridge.bridge.swap; + +/** + * Created by winston on 23/08/2016. + */ +public class SwapJobConfig { + + public static final SwapJobConfig DEFAULT = + new SwapJobConfig(1, 1, 2); + + private final int minProjects; + private final long lowGiB; + private final long highGiB; + + public SwapJobConfig(int minProjects, long lowGiB, long highGiB) { + this.minProjects = minProjects; + this.lowGiB = lowGiB; + this.highGiB = highGiB; + } + + public int getMinProjects() { + return minProjects; + } + + public long getLowGiB() { + return lowGiB; + } + + public long getHighGiB() { + return highGiB; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java index 26f27891b0..f7b179cb4c 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java @@ -14,6 +14,10 @@ import java.util.concurrent.atomic.AtomicInteger; */ public class SwapJobImpl implements SwapJob { + private final int minProjects; + private final long lowGiB; + private final long highGiB; + private final ProjectLock lock; private final RepoStore repoStore; private final SwapStore swapStore; @@ -24,12 +28,15 @@ public class SwapJobImpl implements SwapJob { final AtomicInteger swaps; public SwapJobImpl( + SwapJobConfig cfg, ProjectLock lock, RepoStore repoStore, DBStore dbStore, SwapStore swapStore ) { - + minProjects = cfg.getMinProjects(); + lowGiB = cfg.getLowGiB(); + highGiB = cfg.getHighGiB(); this.lock = lock; this.repoStore = repoStore; this.swapStore = swapStore; @@ -54,7 +61,13 @@ public class SwapJobImpl implements SwapJob { private void doSwap() { Log.info("Running {}th swap", swaps.getAndIncrement()); + while (repoStore.totalSize() > lowGiB) { + doEvict(); + } + } + private void doEvict() { + dbStore.getOldestUnswappedProject(); } } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java index 60ec0a80b7..f62847be6e 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java @@ -13,6 +13,7 @@ import uk.ac.ic.wlgitbridge.bridge.db.DBStore; import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore; import uk.ac.ic.wlgitbridge.bridge.repo.FSRepoStore; import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; +import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobConfig; import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore; import uk.ac.ic.wlgitbridge.git.exception.InvalidRootDirectoryPathException; import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet; @@ -75,7 +76,8 @@ public class GitBridgeServer { bridgeAPI = Bridge.make( repoStore, dbStore, - swapStore + swapStore, + SwapJobConfig.DEFAULT ); jettyServer = new Server(port); configureJettyServer(config); diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java index 464e5b7331..dbd6c17283 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java @@ -44,9 +44,9 @@ public class BridgeTest { repoStore, dbStore, swapStore, + swapJob, snapshotAPI, - resourceCache, - swapJob + resourceCache ); } diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java index 1432b10607..64ab6d2ffa 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java @@ -26,6 +26,17 @@ public class SqliteDBStoreTest { dbStore = new SqliteDBStore(tmpFolder.newFile("dbStore.db")); } + @Test + public void testGetNumProjects() { + assertEquals(0, dbStore.getNumProjects()); + dbStore.setLatestVersionForProject("asdf", 1); + assertEquals(1, dbStore.getNumProjects()); + dbStore.setLatestVersionForProject("asdf1", 2); + assertEquals(2, dbStore.getNumProjects()); + dbStore.setLatestVersionForProject("asdf1", 3); + assertEquals(2, dbStore.getNumProjects()); + } + @Test public void swapTableStartsOutEmpty() { assertNull(dbStore.getOldestUnswappedProject()); diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java index 7114026d00..c8ab595a28 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java @@ -31,6 +31,7 @@ public class SwapJobImplTest { dbStore = mock(DBStore.class); swapStore = mock(SwapStore.class); swapJob = new SwapJobImpl( + SwapJobConfig.DEFAULT, lock, repoStore, dbStore,