github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-04-06 14:20:56 +00:00
Code Issues Projects Releases Packages Wiki Activity

Implement and test SwapJobImpl

Browse source
This commit is contained in:
Winston Li 2016-08-24 15:15:34 +01:00 • committed by Michael Mazour
parent a595acd0a6
commit c459cd57af
33 changed files with 745 additions and 265 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java
View file

@ -4,12 +4,16 @@ import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStoreConfig;
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
import uk.ac.ic.wlgitbridge.util.Instance;
import javax.annotation.Nullable;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Optional;
/**
* Created by Winston on 05/12/14.
@ -36,7 +40,12 @@ public class Config implements JSONSource {
private String apiBaseURL;
private String postbackURL;
private String serviceName;
@Nullable
private Oauth2 oauth2;
@Nullable
private SwapStoreConfig swapStore;
@Nullable
private SwapJobConfig swapJob;
public Config(String configFilePath) throws ConfigFileException,
IOException {
@ -134,6 +143,14 @@ public class Config implements JSONSource {
return oauth2;
}
public Optional<SwapStoreConfig> getSwapStore() {
return Optional.ofNullable(swapStore);
}
public Optional<SwapJobConfig> getSwapJob() {
return Optional.ofNullable(swapJob);
}
private JsonElement getElement(JsonObject configObject, String name) {
JsonElement element = configObject.get(name);
if (element == null) {

16
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java
View file

@ -9,10 +9,9 @@ import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobImpl;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
@ -31,7 +30,6 @@ import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.time.Duration;
import java.util.*;
/**
@ -56,7 +54,7 @@ public class Bridge {
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore,
SwapJobConfig swapJobConfig
Optional<SwapJobConfig> swapJobConfig
) {
ProjectLock lock = new ProjectLockImpl((int threads) ->
Log.info("Waiting for " + threads + " projects...")
@ -66,7 +64,7 @@ public class Bridge {
repoStore,
dbStore,
swapStore,
new SwapJobImpl(
SwapJob.fromConfig(
swapJobConfig,
lock,
repoStore,
@ -108,8 +106,8 @@ public class Bridge {
Log.info("Bye");
}
public void startSwapJob(Duration interval) {
swapJob.start(interval);
public void startSwapJob() {
swapJob.start();
}
/* TODO: Remove these when WLBridged is moved into RepoStore */

2
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java
View file

@ -24,6 +24,8 @@ public interface DBStore {
String getOldestUnswappedProject();
int getNumUnswappedProjects();
/**
* Sets the last accessed time for the given project name.
* @param projectName the project's name

5
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java
View file

@ -86,6 +86,11 @@ public class SqliteDBStore implements DBStore {
return query(new GetOldestProjectName());
}
@Override
public int getNumUnswappedProjects() {
return query(new GetNumUnswappedProjects());
}
@Override
public void setLastAccessedTime(
String projectName,

31
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java Normal file
View file

@ -0,0 +1,31 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by winston on 24/08/2016.
*/
public class GetNumUnswappedProjects implements SQLQuery<Integer> {
private static final String GET_NUM_UNSWAPPED_PROJECTS =
"SELECT COUNT(*)\n" +
" FROM `swap_table`\n" +
" WHERE `last_accessed` IS NOT NULL";
@Override
public String getSQL() {
return GET_NUM_UNSWAPPED_PROJECTS;
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getInt("COUNT(*)");
}
throw new IllegalStateException("Count always returns results");
}
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java Normal file
View file

@ -0,0 +1,10 @@
package uk.ac.ic.wlgitbridge.bridge.lock;
/**
* Created by winston on 24/08/2016.
*/
public interface LockGuard extends AutoCloseable {
void close();
}

8
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java
View file

@ -4,9 +4,17 @@ package uk.ac.ic.wlgitbridge.bridge.lock;
* Created by winston on 20/08/2016.
*/
public interface ProjectLock {
void lockAll();
void lockForProject(String projectName);
void unlockForProject(String projectName);
/* RAII hahaha */
default LockGuard lockGuard(String projectName) {
lockForProject(projectName);
return () -> unlockForProject(projectName);
}
}

7
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java
View file

@ -58,9 +58,12 @@ public class FSRepoStore implements RepoStore {
}
@Override
public InputStream bzip2Project(String projectName) throws IOException {
public InputStream bzip2Project(
String projectName,
long[] sizePtr
) throws IOException {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
return Tar.bz2.zip(getDotGitForProject(projectName));
return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr);
}
@Override

16
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java
View file

@ -25,7 +25,16 @@ public interface RepoStore {
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().
*/
InputStream bzip2Project(String projectName) throws IOException;
InputStream bzip2Project(
String projectName,
long[] sizePtr
) throws IOException;
default InputStream bzip2Project(
String projectName
) throws IOException {
return bzip2Project(projectName, null);
}
void remove(String projectName) throws IOException;
@ -36,6 +45,9 @@ public interface RepoStore {
* @param projectName the name of the project, e.g. abc123
* @param dataStream the data stream containing the bzipped contents.
*/
void unbzip2Project(String projectName, InputStream dataStream) throws IOException;
void unbzip2Project(
String projectName,
InputStream dataStream
) throws IOException;
}

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java
View file

@ -1,14 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import java.time.Duration;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapJob {
void start(Duration interval);
void stop();
}

33
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobConfig.java
View file

@ -1,33 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
/**
* Created by winston on 23/08/2016.
*/
public class SwapJobConfig {
public static final SwapJobConfig DEFAULT =
new SwapJobConfig(1, 1, 2);
private final int minProjects;
private final long lowGiB;
private final long highGiB;
public SwapJobConfig(int minProjects, long lowGiB, long highGiB) {
this.minProjects = minProjects;
this.lowGiB = lowGiB;
this.highGiB = highGiB;
}
public int getMinProjects() {
return minProjects;
}
public long getLowGiB() {
return lowGiB;
}
public long getHighGiB() {
return highGiB;
}
}

73
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java
View file

@ -1,73 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.util.Log;
import java.time.Duration;
import java.util.Timer;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by winston on 20/08/2016.
*/
public class SwapJobImpl implements SwapJob {
private final int minProjects;
private final long lowGiB;
private final long highGiB;
private final ProjectLock lock;
private final RepoStore repoStore;
private final SwapStore swapStore;
private final DBStore dbStore;
private final Timer timer;
final AtomicInteger swaps;
public SwapJobImpl(
SwapJobConfig cfg,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
minProjects = cfg.getMinProjects();
lowGiB = cfg.getLowGiB();
highGiB = cfg.getHighGiB();
this.lock = lock;
this.repoStore = repoStore;
this.swapStore = swapStore;
this.dbStore = dbStore;
timer = new Timer();
swaps = new AtomicInteger(0);
}
@Override
public void start(Duration interval) {
timer.scheduleAtFixedRate(
uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap),
0,
interval.toMillis()
);
}
@Override
public void stop() {
timer.cancel();
}
private void doSwap() {
Log.info("Running {}th swap", swaps.getAndIncrement());
while (repoStore.totalSize() > lowGiB) {
doEvict();
}
}
private void doEvict() {
dbStore.getOldestUnswappedProject();
}
}

21
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java
View file

@ -1,21 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import java.io.IOException;
import java.io.InputStream;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapStore {
void upload(
String projectName,
InputStream uploadStream,
long contentLength
) throws IOException;
InputStream openDownloadStream(String projectName);
void remove(String projectName);
}

30
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java Normal file
View file

@ -0,0 +1,30 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import java.io.IOException;
/**
* Created by winston on 24/08/2016.
*/
public class NoopSwapJob implements SwapJob {
@Override
public void start() {
}
@Override
public void stop() {
}
@Override
public void evict(String projName) throws IOException {
}
@Override
public void restore(String projName) throws IOException {
}
}

42
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java Normal file
View file

@ -0,0 +1,42 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import java.io.IOException;
import java.util.Optional;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapJob {
static SwapJob fromConfig(
Optional<SwapJobConfig> cfg,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
if (cfg.isPresent()) {
return new SwapJobImpl(
cfg.get(),
lock,
repoStore,
dbStore,
swapStore
);
}
return new NoopSwapJob();
}
void start();
void stop();
void evict(String projName) throws IOException;
void restore(String projName) throws IOException;
}

44
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java Normal file
View file

@ -0,0 +1,44 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
/**
* Created by winston on 23/08/2016.
*/
public class SwapJobConfig {
public static final SwapJobConfig DEFAULT =
new SwapJobConfig(1, 1, 2, 3600000);
private final int minProjects;
private final int lowGiB;
private final int highGiB;
private final long intervalMillis;
public SwapJobConfig(
int minProjects,
int lowGiB,
int highGiB,
long intervalMillis
) {
this.minProjects = minProjects;
this.lowGiB = lowGiB;
this.highGiB = highGiB;
this.intervalMillis = intervalMillis;
}
public int getMinProjects() {
return minProjects;
}
public int getLowGiB() {
return lowGiB;
}
public int getHighGiB() {
return highGiB;
}
public long getIntervalMillis() {
return intervalMillis;
}
}

163
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java Normal file
View file

@ -0,0 +1,163 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Timer;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by winston on 20/08/2016.
*/
public class SwapJobImpl implements SwapJob {
private static final long GiB = (1l << 30);
int minProjects;
long lowWatermarkBytes;
long highWatermarkBytes;
Duration interval;
private final ProjectLock lock;
private final RepoStore repoStore;
private final DBStore dbStore;
private final SwapStore swapStore;
private final Timer timer;
final AtomicInteger swaps;
public SwapJobImpl(
SwapJobConfig cfg,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
this(
cfg.getMinProjects(),
GiB * cfg.getLowGiB(),
GiB * cfg.getHighGiB(),
Duration.ofMillis(cfg.getIntervalMillis()),
lock,
repoStore,
dbStore,
swapStore
);
}
SwapJobImpl(
int minProjects,
long lowWatermarkBytes,
long highWatermarkBytes,
Duration interval,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
this.minProjects = minProjects;
this.lowWatermarkBytes = lowWatermarkBytes;
this.highWatermarkBytes = highWatermarkBytes;
this.interval = interval;
this.lock = lock;
this.repoStore = repoStore;
this.dbStore = dbStore;
this.swapStore = swapStore;
timer = new Timer();
swaps = new AtomicInteger(0);
}
@Override
public void start() {
timer.scheduleAtFixedRate(
uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap),
0,
interval.toMillis()
);
}
@Override
public void stop() {
timer.cancel();
}
private void doSwap() {
Log.info("Running swap number {}", swaps.get() + 1);
long totalSize = repoStore.totalSize();
Log.info("Size is {}/{} (high)", totalSize, highWatermarkBytes);
if (totalSize < highWatermarkBytes) {
Log.info("No need to swap.");
swaps.incrementAndGet();
return;
}
int numProjects = dbStore.getNumProjects();
while (
(totalSize = repoStore.totalSize()) > lowWatermarkBytes &&
(numProjects = dbStore.getNumUnswappedProjects()) > minProjects
) {
try {
evict(dbStore.getOldestUnswappedProject());
} catch (IOException e) {
Log.warn("Exception while swapping, giving up", e);
}
}
if (totalSize > lowWatermarkBytes) {
Log.warn(
"Finished swapping, but total size is still too high."
);
}
Log.info(
"Size: {}/{} (low), " +
"{} (high), " +
"projects on disk: {}/{}, " +
"min projects on disk: {}",
totalSize,
lowWatermarkBytes,
highWatermarkBytes,
numProjects,
dbStore.getNumProjects(),
minProjects
);
swaps.incrementAndGet();
}
@Override
public void evict(String projName) throws IOException {
Preconditions.checkNotNull(projName);
Log.info("Evicting project: {}", projName);
try (LockGuard __ = lock.lockGuard(projName)) {
long[] sizePtr = new long[1];
InputStream bzipped = repoStore.bzip2Project(projName, sizePtr);
swapStore.upload(projName, bzipped, sizePtr[0]);
dbStore.setLastAccessedTime(projName, null);
repoStore.remove(projName);
}
Log.info("Evicted project: {}", projName);
}
@Override
public void restore(String projName) throws IOException {
try (LockGuard __ = lock.lockGuard(projName)) {
repoStore.unbzip2Project(
projName,
swapStore.openDownloadStream(projName)
);
dbStore.setLastAccessedTime(
projName,
Timestamp.valueOf(LocalDateTime.now())
);
}
}
}

2
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/InMemorySwapStore.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStore.java
View file

@ -1,4 +1,4 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import org.apache.commons.io.IOUtils;

35
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java Normal file
View file

@ -0,0 +1,35 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
/**
* Created by winston on 24/08/2016.
*/
public class NoopSwapStore implements SwapStore {
public NoopSwapStore(SwapStoreConfig config) {
}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) throws IOException {
}
@Override
public InputStream openDownloadStream(String projectName) {
return new ByteArrayInputStream(new byte[0]);
}
@Override
public void remove(String projectName) {
}
}

12
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStore.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStore.java
View file

@ -1,4 +1,4 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
@ -16,7 +16,15 @@ public class S3SwapStore implements SwapStore {
private final String bucketName;
public S3SwapStore(
public S3SwapStore(SwapStoreConfig cfg) {
this(
cfg.getAwsAccessKey(),
cfg.getAwsSecret(),
cfg.getS3BucketName()
);
}
S3SwapStore(
String accessKey,
String secret,
String bucketName

43
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java Normal file
View file

@ -0,0 +1,43 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapStore {
Map<String, Function<SwapStoreConfig, SwapStore>> swapStores =
new HashMap<String, Function<SwapStoreConfig, SwapStore>>() {
{
put("noop", NoopSwapStore::new);
put("s3", S3SwapStore::new);
}
};
static SwapStore fromConfig(
Optional<SwapStoreConfig> cfg
) {
SwapStoreConfig cfg_ = cfg.orElse(SwapStoreConfig.NOOP);
String type = cfg_.getType();
return swapStores.get(type).apply(cfg_);
}
void upload(
String projectName,
InputStream uploadStream,
long contentLength
) throws IOException;
InputStream openDownloadStream(String projectName);
void remove(String projectName);
}

63
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java Normal file
View file

@ -0,0 +1,63 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
/**
* Created by winston on 24/08/2016.
*/
public class SwapStoreConfig {
public static final SwapStoreConfig NOOP = new SwapStoreConfig(
"noop",
null,
null,
null
);
private String type;
private String awsAccessKey;
private String awsSecret;
private String s3BucketName;
public SwapStoreConfig() {}
public SwapStoreConfig(
String awsAccessKey,
String awsSecret,
String s3BucketName
) {
this(
"s3",
awsAccessKey,
awsSecret,
s3BucketName
);
}
SwapStoreConfig(
String type,
String awsAccessKey,
String awsSecret,
String s3BucketName
) {
this.type = type;
this.awsAccessKey = awsAccessKey;
this.awsSecret = awsSecret;
this.s3BucketName = s3BucketName;
}
public String getType() {
return type;
}
public String getAwsAccessKey() {
return awsAccessKey;
}
public String getAwsSecret() {
return awsSecret;
}
public String getS3BucketName() {
return s3BucketName;
}
}

60
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
View file

@ -13,8 +13,7 @@ import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore;
import uk.ac.ic.wlgitbridge.bridge.repo.FSRepoStore;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.git.exception.InvalidRootDirectoryPathException;
import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
import uk.ac.ic.wlgitbridge.snapshot.base.SnapshotAPIRequest;
@ -25,7 +24,6 @@ import javax.servlet.DispatcherType;
import javax.servlet.Filter;
import javax.servlet.ServletException;
import java.io.File;
import java.io.InputStream;
import java.net.BindException;
import java.nio.file.Paths;
import java.util.EnumSet;
@ -47,7 +45,9 @@ public class GitBridgeServer {
private String rootGitDirectoryPath;
private String apiBaseURL;
public GitBridgeServer(Config config) throws ServletException, InvalidRootDirectoryPathException {
public GitBridgeServer(
Config config
) throws ServletException, InvalidRootDirectoryPathException {
org.eclipse.jetty.util.log.Log.setLog(new NullLogger());
this.port = config.getPort();
this.rootGitDirectoryPath = config.getRootGitDirectory();
@ -57,31 +57,19 @@ public class GitBridgeServer {
repoStore.getRootDirectory().getAbsolutePath()
).resolve(".wlgb").resolve("wlgb.db").toFile()
);
SwapStore swapStore = new SwapStore() {
@Override
public void upload(String projectName, InputStream uploadStream, long contentLength) {
}
@Override
public InputStream openDownloadStream(String projectName) {
return null;
}
@Override
public void remove(String projectName) {
}
};
SwapStore swapStore = SwapStore.fromConfig(config.getSwapStore());
bridgeAPI = Bridge.make(
repoStore,
dbStore,
swapStore,
SwapJobConfig.DEFAULT
config.getSwapJob()
);
jettyServer = new Server(port);
configureJettyServer(config);
SnapshotAPIRequest.setBasicAuth(config.getUsername(), config.getPassword());
SnapshotAPIRequest.setBasicAuth(
config.getUsername(),
config.getPassword()
);
apiBaseURL = config.getAPIBaseURL();
SnapshotAPIRequest.setBaseURL(apiBaseURL);
Util.setServiceName(config.getServiceName());
@ -115,7 +103,9 @@ public class GitBridgeServer {
}
}
private void configureJettyServer(Config config) throws ServletException, InvalidRootDirectoryPathException {
private void configureJettyServer(
Config config
) throws ServletException, InvalidRootDirectoryPathException {
HandlerCollection handlers = new HandlerList();
handlers.addHandler(initApiHandler());
handlers.addHandler(initGitHandler(config));
@ -135,16 +125,28 @@ public class GitBridgeServer {
return api;
}
private Handler initGitHandler(Config config) throws ServletException, InvalidRootDirectoryPathException {
final ServletContextHandler servletContextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS);
private Handler initGitHandler(
Config config
) throws ServletException, InvalidRootDirectoryPathException {
final ServletContextHandler servletContextHandler =
new ServletContextHandler(ServletContextHandler.SESSIONS);
if (config.isUsingOauth2()) {
Filter filter = new Oauth2Filter(config.getOauth2());
servletContextHandler.addFilter(new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST));
servletContextHandler.addFilter(
new FilterHolder(filter),
"/*",
EnumSet.of(DispatcherType.REQUEST)
);
}
servletContextHandler.setContextPath("/");
servletContextHandler.addServlet(
new ServletHolder(
new WLGitServlet(servletContextHandler, bridgeAPI, rootGitDirectoryPath)),
new WLGitServlet(
servletContextHandler,
bridgeAPI,
rootGitDirectoryPath
)
),
"/*"
);
return servletContextHandler;
@ -152,7 +154,9 @@ public class GitBridgeServer {
private Handler initResourceHandler() {
ResourceHandler resourceHandler = new FileHandler(bridgeAPI);
resourceHandler.setResourceBase(new File(rootGitDirectoryPath, ".wlgb/atts").getAbsolutePath());
resourceHandler.setResourceBase(
new File(rootGitDirectoryPath, ".wlgb/atts").getAbsolutePath()
);
return resourceHandler;
}
}

11
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java
View file

@ -23,11 +23,21 @@ public class Tar {
public static InputStream zip(
File fileOrDir
) throws IOException {
return zip(fileOrDir, null);
}
public static InputStream zip(
File fileOrDir,
long[] sizePtr
) throws IOException {
ByteArrayOutputStream target = new ByteArrayOutputStream();
try (OutputStream bzip2 = new BZip2CompressorOutputStream(target)) {
tarTo(fileOrDir, bzip2);
}
if (sizePtr != null) {
sizePtr[0] = target.size();
}
return target.toInputStream();
}
@ -60,7 +70,6 @@ public class Tar {
Paths.get(fileOrDir.getParentFile().getAbsolutePath()),
fileOrDir
);
tout.close();
}
}

10
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
View file

@ -7,10 +7,8 @@ import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import java.time.Duration;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
@ -52,9 +50,9 @@ public class BridgeTest {
@Test
public void shutdownStopsSwapJob() {
bridge.startSwapJob(Duration.ofSeconds(1));
bridge.startSwapJob();
bridge.doShutdown();
verify(swapJob).start(Duration.ofSeconds(1));
verify(swapJob).start();
verify(swapJob).stop();
}

15
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java
View file

@ -132,4 +132,19 @@ public class SqliteDBStoreTest {
assertEquals("older", dbStore.getOldestUnswappedProject());
}
@Test
public void testGetNumUnswappedProjects() {
dbStore.setLatestVersionForProject("asdf", 1);
dbStore.setLastAccessedTime(
"asdf",
Timestamp.valueOf(LocalDateTime.now())
);
assertEquals(1, dbStore.getNumUnswappedProjects());
dbStore.setLastAccessedTime(
"asdf",
null
);
assertEquals(0, dbStore.getNumUnswappedProjects());
}
}

14
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdateTest.java
View file

@ -7,8 +7,18 @@ public class DeleteFilesForProjectSQLUpdateTest {
@Test
public void testGetSQL() {
DeleteFilesForProjectSQLUpdate update = new DeleteFilesForProjectSQLUpdate("projname", "path1", "path2");
assertEquals("DELETE FROM `url_index_store` WHERE `project_name` = ? AND path IN (?, ?);\n", update.getSQL());
DeleteFilesForProjectSQLUpdate update =
new DeleteFilesForProjectSQLUpdate(
"projname",
"path1",
"path2"
);
assertEquals(
"DELETE FROM `url_index_store` " +
"WHERE `project_name` = ? " +
"AND path IN (?, ?);\n",
update.getSQL()
);
}
}

22
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest.java
View file

@ -21,6 +21,20 @@ import static org.junit.Assert.assertEquals;
*/
public class FSRepoStoreTest {
public static File makeTempRepoDir(
TemporaryFolder tmpFolder,
String name
) throws IOException {
File tmp = tmpFolder.newFolder(name);
Path rootdir = Paths.get(
"src/test/resources/uk/ac/ic/wlgitbridge/"
+ "bridge/repo/FSRepoStoreTest/rootdir"
);
FileUtils.copyDirectory(rootdir.toFile(), tmp);
Files.renameAll(tmp, "DOTgit", ".git");
return tmp;
}
private FSRepoStore repoStore;
private File original;
@ -28,13 +42,7 @@ public class FSRepoStoreTest {
public void setup() throws IOException {
TemporaryFolder tmpFolder = new TemporaryFolder();
tmpFolder.create();
File tmp = tmpFolder.newFolder("repostore");
Path rootdir = Paths.get(
"src/test/resources/uk/ac/ic/wlgitbridge/"
+ "bridge/repo/FSRepoStoreTest/rootdir"
);
FileUtils.copyDirectory(rootdir.toFile(), tmp);
Files.renameAll(tmp, "DOTgit", ".git");
File tmp = makeTempRepoDir(tmpFolder, "rootdir");
original = tmpFolder.newFolder("original");
FileUtils.copyDirectory(tmp, original);
repoStore = new FSRepoStore(tmp.getAbsolutePath());

58
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java
View file

@ -1,58 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import org.junit.Before;
import org.junit.Test;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import java.time.Duration;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
/**
* Created by winston on 20/08/2016.
*/
public class SwapJobImplTest {
private SwapJobImpl swapJob;
private ProjectLock lock;
private RepoStore repoStore;
private DBStore dbStore;
private SwapStore swapStore;
@Before
public void setup() {
lock = mock(ProjectLock.class);
repoStore = mock(RepoStore.class);
dbStore = mock(DBStore.class);
swapStore = mock(SwapStore.class);
swapJob = new SwapJobImpl(
SwapJobConfig.DEFAULT,
lock,
repoStore,
dbStore,
swapStore
);
}
@Test
public void startingTimerAlwaysCausesASwap() {
assertEquals(0, swapJob.swaps.get());
swapJob.start(Duration.ofHours(1));
while (swapJob.swaps.get() <= 0);
assertTrue(swapJob.swaps.get() > 0);
}
@Test
public void swapsHappenEveryInterval() {
assertEquals(0, swapJob.swaps.get());
swapJob.start(Duration.ofMillis(1));
while (swapJob.swaps.get() <= 1);
assertTrue(swapJob.swaps.get() > 1);
}
}

119
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java Normal file
View file

@ -0,0 +1,119 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import org.junit.Before;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.FSRepoStore;
import uk.ac.ic.wlgitbridge.bridge.repo.FSRepoStoreTest;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.InMemorySwapStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
import java.io.IOException;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Created by winston on 20/08/2016.
*/
public class SwapJobImplTest {
private SwapJobImpl swapJob;
private ProjectLock lock;
private RepoStore repoStore;
private DBStore dbStore;
private SwapStore swapStore;
@Before
public void setup() throws IOException {
TemporaryFolder tmpFolder = new TemporaryFolder();
tmpFolder.create();
lock = new ProjectLockImpl();
repoStore = new FSRepoStore(
FSRepoStoreTest.makeTempRepoDir(
tmpFolder,
"repostore"
).getAbsolutePath()
);
dbStore = new SqliteDBStore(tmpFolder.newFile());
dbStore.setLatestVersionForProject("proj1", 0);
dbStore.setLatestVersionForProject("proj2", 0);
dbStore.setLastAccessedTime(
"proj1",
Timestamp.valueOf(LocalDateTime.now())
);
dbStore.setLastAccessedTime(
"proj2",
Timestamp.valueOf(
LocalDateTime.now().minus(1, ChronoUnit.SECONDS)
)
);
swapStore = new InMemorySwapStore();
swapJob = new SwapJobImpl(
1,
15000,
30000,
Duration.ofMillis(100),
lock,
repoStore,
dbStore,
swapStore
);
}
@Test
public void startingTimerAlwaysCausesASwap() {
swapJob.lowWatermarkBytes = 16384;
swapJob.interval = Duration.ofHours(1);
assertEquals(0, swapJob.swaps.get());
swapJob.start();
while (swapJob.swaps.get() <= 0);
assertTrue(swapJob.swaps.get() > 0);
}
@Test
public void swapsHappenEveryInterval() {
swapJob.lowWatermarkBytes = 16384;
assertEquals(0, swapJob.swaps.get());
swapJob.start();
while (swapJob.swaps.get() <= 1);
assertTrue(swapJob.swaps.get() > 1);
}
@Test
public void noProjectsGetSwappedWhenUnderHighWatermark() {
swapJob.highWatermarkBytes = 65536;
assertEquals(2, dbStore.getNumUnswappedProjects());
swapJob.start();
while (swapJob.swaps.get() < 1);
assertEquals(2, dbStore.getNumUnswappedProjects());
}
@Test
public void correctProjGetSwappedWhenOverHighWatermark(
) throws IOException {
swapJob.lowWatermarkBytes = 16384;
assertEquals(2, dbStore.getNumUnswappedProjects());
assertEquals("proj2", dbStore.getOldestUnswappedProject());
swapJob.start();
while (swapJob.swaps.get() < 1);
assertEquals(1, dbStore.getNumUnswappedProjects());
assertEquals("proj1", dbStore.getOldestUnswappedProject());
swapJob.restore("proj2");
int numSwaps = swapJob.swaps.get();
while (swapJob.swaps.get() <= numSwaps);
assertEquals(1, dbStore.getNumUnswappedProjects());
assertEquals("proj2", dbStore.getOldestUnswappedProject());
}
}

6
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/InMemorySwapStoreTest.java → services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStoreTest.java
View file

@ -1,9 +1,10 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import org.apache.commons.io.IOUtils;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import uk.ac.ic.wlgitbridge.bridge.swap.store.InMemorySwapStore;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@ -51,7 +52,8 @@ public class InMemorySwapStoreTest {
}
@Test
public void uploadingForTheSameProjectOverwritesTheFile() throws IOException {
public void uploadingForTheSameProjectOverwritesTheFile(
) throws IOException {
byte[] proj1Contents = "helloproj1".getBytes();
byte[] proj1NewContents = "goodbyeproj1".getBytes();
swapStore.upload(

2
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStoreTest.java → services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStoreTest.java
View file

@ -1,4 +1,4 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import org.junit.Before;

6
services/git-bridge/src/test/resources/logback-test.xml
View file

@ -1,7 +1,7 @@
<configuration>
<!-- Log everything (subject to logger and root levels set below) to stdout. -->
<appender name="tempfile" class="ch.qos.logback.core.FileAppender">
<file>${java.io.tmpdir}/git-bridge-test.log</file>
<appender name="stderr" class="ch.qos.logback.core.ConsoleAppender">
<target>System.err</target>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] %logger{0}: %msg%n</pattern>
</encoder>
@ -12,6 +12,6 @@
<!-- The root log level determines how much our dependencies put in the logs. -->
<root level="WARN">
<appender-ref ref="tempfile" />
<appender-ref ref="stderr" />
</root>
</configuration>