diff --git a/services/git-bridge/Dockerfile b/services/git-bridge/Dockerfile index 9de0342f0b..5367380bde 100644 --- a/services/git-bridge/Dockerfile +++ b/services/git-bridge/Dockerfile @@ -2,7 +2,7 @@ FROM maven:3-jdk-11 as base -RUN apt-get update && apt-get install -y make git \ +RUN apt-get update && apt-get install -y make git sqlite3 \ && rm -rf /var/lib/apt/lists COPY vendor/envsubst /opt/envsubst diff --git a/services/git-bridge/conf/envsubst_template.json b/services/git-bridge/conf/envsubst_template.json index 5d3c28affc..df03c9939a 100644 --- a/services/git-bridge/conf/envsubst_template.json +++ b/services/git-bridge/conf/envsubst_template.json @@ -28,5 +28,6 @@ "highGiB": ${GIT_BRIDGE_SWAPJOB_HIGH_GIB:-256}, "intervalMillis": ${GIT_BRIDGE_SWAPJOB_INTERVAL_MILLIS:-3600000}, "compressionMethod": "${GIT_BRIDGE_SWAPJOB_COMPRESSION_METHOD:-gzip}" - } + }, + "sqliteHeapLimitBytes": ${GIT_BRIDGE_SQLITE_HEAP_LIMIT_BYTES:-0} } diff --git a/services/git-bridge/conf/example_config.json b/services/git-bridge/conf/example_config.json index d2d1e3803e..3972be2cb9 100644 --- a/services/git-bridge/conf/example_config.json +++ b/services/git-bridge/conf/example_config.json @@ -28,5 +28,6 @@ "highGiB": 256, "intervalMillis": 3600000, "compressionMethod": "gzip" - } + }, + "sqliteHeapLimitBytes": 512000000 } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java index 8b1bb19da1..8dbef10b23 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java @@ -33,7 +33,8 @@ public class Config implements JSONSource { Oauth2.asSanitised(config.oauth2), config.repoStore, SwapStoreConfig.sanitisedCopy(config.swapStore), - config.swapJob + config.swapJob, + config.sqliteHeapLimitBytes ); } @@ -52,6 +53,7 @@ public class Config implements JSONSource { private SwapStoreConfig swapStore; @Nullable private SwapJobConfig swapJob; + private int sqliteHeapLimitBytes = 0; public Config( String configFilePath @@ -75,7 +77,8 @@ public class Config implements JSONSource { Oauth2 oauth2, RepoStoreConfig repoStore, SwapStoreConfig swapStore, - SwapJobConfig swapJob + SwapJobConfig swapJob, + int sqliteHeapLimitBytes ) { this.port = port; this.bindIp = bindIp; @@ -88,6 +91,7 @@ public class Config implements JSONSource { this.repoStore = repoStore; this.swapStore = swapStore; this.swapJob = swapJob; + this.sqliteHeapLimitBytes = sqliteHeapLimitBytes; } @Override @@ -124,6 +128,9 @@ public class Config implements JSONSource { configObject.get("swapJob"), SwapJobConfig.class ); + if (configObject.has("sqliteHeapLimitBytes")) { + sqliteHeapLimitBytes = getElement(configObject, "sqliteHeapLimitBytes").getAsInt(); + } } public String getSanitisedString() { @@ -146,6 +153,10 @@ public class Config implements JSONSource { return rootGitDirectory; } + public int getSqliteHeapLimitBytes() { + return this.sqliteHeapLimitBytes; + } + public String getAPIBaseURL() { return apiBaseURL; } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java index 725347bc0d..ec59313bae 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java @@ -21,8 +21,14 @@ import java.util.stream.Stream; public class SqliteDBStore implements DBStore { private final Connection connection; + private int heapLimitBytes = 0; public SqliteDBStore(File dbFile) { + this(dbFile, 0); + } + + public SqliteDBStore(File dbFile, int heapLimitBytes) { + this.heapLimitBytes = heapLimitBytes; try { connection = openConnectionTo(dbFile); createTables(); @@ -144,6 +150,7 @@ public class SqliteDBStore implements DBStore { private void createTables() { /* Migrations */ /* We need to eat exceptions from here */ + try { doUpdate(new SetSoftHeapLimitPragma(this.heapLimitBytes)); } catch (SQLException ignore) {} try { doUpdate(new ProjectsAddLastAccessed()); } catch (SQLException ignore) {} try { doUpdate(new ProjectsAddSwapTime()); } catch (SQLException ignore) {} try { doUpdate(new ProjectsAddRestoreTime()); } catch (SQLException ignore) {} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/SetSoftHeapLimitPragma.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/SetSoftHeapLimitPragma.java new file mode 100644 index 0000000000..da1a98cfa5 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/SetSoftHeapLimitPragma.java @@ -0,0 +1,17 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; + +public class SetSoftHeapLimitPragma implements SQLUpdate { + private int heapLimitBytes = 0; + + public SetSoftHeapLimitPragma(int heapLimitBytes) { + this.heapLimitBytes = heapLimitBytes; + } + + @Override + public String getSQL() { + return "PRAGMA soft_heap_limit="+this.heapLimitBytes+";"; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java index 04980d9235..312e3648b9 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java @@ -61,7 +61,8 @@ public class GitBridgeServer { DBStore dbStore = new SqliteDBStore( Paths.get( repoStore.getRootDirectory().getAbsolutePath() - ).resolve(".wlgb").resolve("wlgb.db").toFile() + ).resolve(".wlgb").resolve("wlgb.db").toFile(), + config.getSqliteHeapLimitBytes() ); SwapStore swapStore = SwapStore.fromConfig(config.getSwapStore()); SnapshotApi snapshotApi = new NetSnapshotApi(); diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java index 4710dcff6f..02d70def0e 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java @@ -94,7 +94,8 @@ public class ConfigTest { " },\n" + " \"repoStore\": null,\n" + " \"swapStore\": null,\n" + - " \"swapJob\": null\n" + + " \"swapJob\": null,\n" + + " \"sqliteHeapLimitBytes\": 0\n" + "}"; assertEquals( "sanitised config did not hide sensitive fields", diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java index 6a3fe4eb08..9026973d49 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java @@ -63,7 +63,8 @@ public class BridgeTest { null, null, null, - null), + null, + 0), lock, repoStore, dbStore,