Implement and test the swap job, and add integration test

This commit is contained in:
Winston Li 2016-08-24 18:04:01 +01:00 committed by Michael Mazour
parent dd5694104d
commit 9936fbe3c9
20 changed files with 258 additions and 45 deletions

View file

@ -25,7 +25,7 @@ public class GitBridgeApp implements Runnable {
"usage: writelatex-git-bridge config_file";
private String configFilePath;
private Config config;
Config config;
private GitBridgeServer server;
/**

View file

@ -98,6 +98,14 @@ public class Config implements JSONSource {
postbackURL += "/";
}
oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
swapStore = new Gson().fromJson(
configObject.get("swapStore"),
SwapStoreConfig.class
);
swapJob = new Gson().fromJson(
configObject.get("swapJob"),
SwapJobConfig.class
);
}
public String getSanitisedString() {

View file

@ -10,6 +10,9 @@ import java.util.Collection;
*/
public interface RepoStore {
/* Still need to get rid of these two methods.
Main dependency: GitRepoStore needs a Repository which needs a directory.
Instead, use a visitor or something. */
String getRepoStorePath();
File getRootDirectory();
@ -20,7 +23,7 @@ public interface RepoStore {
long totalSize();
/*
/**
* Tars and bzip2s the .git directory of the given project. Throws an
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().

View file

@ -5,9 +5,6 @@ package uk.ac.ic.wlgitbridge.bridge.swap.job;
*/
public class SwapJobConfig {
public static final SwapJobConfig DEFAULT =
new SwapJobConfig(1, 1, 2, 3600000);
private final int minProjects;
private final int lowGiB;
private final int highGiB;

View file

@ -19,6 +19,10 @@ public class InMemorySwapStore implements SwapStore {
store = new HashMap<>();
}
public InMemorySwapStore(SwapStoreConfig __) {
this();
}
@Override
public void upload(
String projectName,

View file

@ -17,6 +17,7 @@ public interface SwapStore {
{
put("noop", NoopSwapStore::new);
put("memory", InMemorySwapStore::new);
put("s3", S3SwapStore::new);
}

View file

@ -19,10 +19,10 @@ import javax.servlet.http.HttpServletRequest;
/* */
public class WLReceivePackFactory implements ReceivePackFactory<HttpServletRequest> {
private final Bridge bridgeAPI;
private final Bridge bridge;
public WLReceivePackFactory(Bridge bridgeAPI) {
this.bridgeAPI = bridgeAPI;
public WLReceivePackFactory(Bridge bridge) {
this.bridge = bridge;
}
@Override
@ -33,7 +33,7 @@ public class WLReceivePackFactory implements ReceivePackFactory<HttpServletReque
if (hostname == null) {
hostname = httpServletRequest.getLocalName();
}
receivePack.setPreReceiveHook(new WriteLatexPutHook(bridgeAPI, hostname, oauth2));
receivePack.setPreReceiveHook(new WriteLatexPutHook(bridge, hostname, oauth2));
return receivePack;
}

View file

@ -26,12 +26,12 @@ import java.util.Iterator;
*/
public class WriteLatexPutHook implements PreReceiveHook {
private final Bridge bridgeAPI;
private final Bridge bridge;
private final String hostname;
private final Credential oauth2;
public WriteLatexPutHook(Bridge bridgeAPI, String hostname, Credential oauth2) {
this.bridgeAPI = bridgeAPI;
public WriteLatexPutHook(Bridge bridge, String hostname, Credential oauth2) {
this.bridge = bridge;
this.hostname = hostname;
this.oauth2 = oauth2;
}
@ -75,7 +75,7 @@ public class WriteLatexPutHook implements PreReceiveHook {
private void handleReceiveCommand(Credential oauth2, Repository repository, ReceiveCommand receiveCommand) throws IOException, GitUserException {
checkBranch(receiveCommand);
checkForcedPush(receiveCommand);
bridgeAPI.putDirectoryContentsToProjectWithName(
bridge.putDirectoryContentsToProjectWithName(
oauth2,
repository.getWorkTree().getName(),
getPushedDirectoryContents(repository,

View file

@ -37,7 +37,7 @@ import java.util.EnumSet;
*/
public class GitBridgeServer {
private final Bridge bridgeAPI;
private final Bridge bridge;
private final Server jettyServer;
@ -58,7 +58,7 @@ public class GitBridgeServer {
).resolve(".wlgb").resolve("wlgb.db").toFile()
);
SwapStore swapStore = SwapStore.fromConfig(config.getSwapStore());
bridgeAPI = Bridge.make(
bridge = Bridge.make(
repoStore,
dbStore,
swapStore,
@ -83,6 +83,7 @@ public class GitBridgeServer {
public void start() {
try {
jettyServer.start();
bridge.startSwapJob();
Log.info(Util.getServiceName() + "-Git Bridge server started");
Log.info("Listening on port: " + port);
Log.info("Bridged to: " + apiBaseURL);
@ -118,7 +119,7 @@ public class GitBridgeServer {
HandlerCollection handlers = new HandlerList();
handlers.addHandler(initResourceHandler());
handlers.addHandler(new PostbackHandler(bridgeAPI));
handlers.addHandler(new PostbackHandler(bridge));
handlers.addHandler(new DefaultHandler());
api.setHandler(handlers);
@ -143,7 +144,7 @@ public class GitBridgeServer {
new ServletHolder(
new WLGitServlet(
servletContextHandler,
bridgeAPI
bridge
)
),
"/*"
@ -152,7 +153,7 @@ public class GitBridgeServer {
}
private Handler initResourceHandler() {
ResourceHandler resourceHandler = new FileHandler(bridgeAPI);
ResourceHandler resourceHandler = new FileHandler(bridge);
resourceHandler.setResourceBase(
new File(rootGitDirectoryPath, ".wlgb/atts").getAbsolutePath()
);

View file

@ -17,7 +17,7 @@ public class PostbackContents implements JSONSource {
private static final String CODE_SUCCESS = "upToDate";
private final Bridge bridgeAPI;
private final Bridge bridge;
private final String projectName;
private final String postbackKey;
@ -26,8 +26,8 @@ public class PostbackContents implements JSONSource {
private int versionID;
private SnapshotPostException exception;
public PostbackContents(Bridge bridgeAPI, String projectName, String postbackKey, String contents) {
this.bridgeAPI = bridgeAPI;
public PostbackContents(Bridge bridge, String projectName, String postbackKey, String contents) {
this.bridge = bridge;
this.projectName = projectName;
this.postbackKey = postbackKey;
snapshotPostExceptionBuilder = new SnapshotPostExceptionBuilder();
@ -43,9 +43,9 @@ public class PostbackContents implements JSONSource {
public void processPostback() throws UnexpectedPostbackException {
if (exception == null) {
bridgeAPI.postbackReceivedSuccessfully(projectName, postbackKey, versionID);
bridge.postbackReceivedSuccessfully(projectName, postbackKey, versionID);
} else {
bridgeAPI.postbackReceivedWithException(projectName, postbackKey, exception);
bridge.postbackReceivedWithException(projectName, postbackKey, exception);
}
}

View file

@ -19,10 +19,10 @@ import java.io.IOException;
*/
public class PostbackHandler extends AbstractHandler {
private final Bridge bridgeAPI;
private final Bridge bridge;
public PostbackHandler(Bridge bridgeAPI) {
this.bridgeAPI = bridgeAPI;
public PostbackHandler(Bridge bridge) {
this.bridge = bridge;
}
@Override
@ -39,7 +39,7 @@ public class PostbackHandler extends AbstractHandler {
String projectName = parts[1];
String postbackKey = parts[2];
Log.info(baseRequest.getMethod() + " <- " + baseRequest.getHttpURI());
PostbackContents postbackContents = new PostbackContents(bridgeAPI, projectName, postbackKey, contents);
PostbackContents postbackContents = new PostbackContents(bridge, projectName, postbackKey, contents);
JsonObject body = new JsonObject();
try {

View file

@ -1,13 +1,14 @@
package uk.ac.ic.wlgitbridge;
package uk.ac.ic.wlgitbridge.application;
import com.ning.http.client.AsyncHttpClient;
import com.ning.http.client.Response;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import uk.ac.ic.wlgitbridge.application.GitBridgeApp;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
import uk.ac.ic.wlgitbridge.snapshot.servermock.server.MockSnapshotServer;
import uk.ac.ic.wlgitbridge.snapshot.servermock.state.SnapshotAPIState;
import uk.ac.ic.wlgitbridge.snapshot.servermock.state.SnapshotAPIStateBuilder;
@ -102,6 +103,9 @@ public class WLGitBridgeIntegrationTest {
put("canServePushedFiles", new HashMap<String, SnapshotAPIState>() {{
put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/canServePushedFiles/state/state.json")).build());
}});
put("wlgbCanSwapProjects", new HashMap<String, SnapshotAPIState>() {{
put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/wlgbCanSwapProjects/state/state.json")).build());
}});
}};
@Rule
@ -587,6 +591,38 @@ public class WLGitBridgeIntegrationTest {
wlgb.stop();
}
@Test
public void wlgbCanSwapProjects(
) throws IOException, GitAPIException, InterruptedException {
MockSnapshotServer server = new MockSnapshotServer(
3874,
getResource("/wlgbCanSwapProjects").toFile()
);
server.start();
server.setState(states.get("wlgbCanSwapProjects").get("state"));
GitBridgeApp wlgb = new GitBridgeApp(new String[] {
makeConfigFile(33874, 3874, new SwapJobConfig(1, 0, 0, 250))
});
wlgb.run();
File dir = folder.newFolder();
File rootGitDir = new File(wlgb.config.getRootGitDirectory());
File testProj1ServerDir = new File(rootGitDir, "testproj1");
File testProj2ServerDir = new File(rootGitDir, "testproj2");
File testProj1Dir = cloneRepository("testproj1", 33874, dir);
assertTrue(testProj1ServerDir.exists());
assertFalse(testProj2ServerDir.exists());
cloneRepository("testproj2", 33874, dir);
while (testProj1ServerDir.exists());
assertFalse(testProj1ServerDir.exists());
assertTrue(testProj2ServerDir.exists());
FileUtils.deleteDirectory(testProj1Dir);
cloneRepository("testproj1", 33874, dir);
while (testProj2ServerDir.exists());
assertTrue(testProj1ServerDir.exists());
assertFalse(testProj2ServerDir.exists());
wlgb.stop();
}
private File cloneRepository(String repositoryName, int port, File dir) throws IOException, InterruptedException {
String repo = "git clone http://127.0.0.1:" + port + "/" + repositoryName + ".git";
Process gitProcess = runtime.exec(repo, null, dir);
@ -606,30 +642,72 @@ public class WLGitBridgeIntegrationTest {
return repositoryDir;
}
private String makeConfigFile(int port, int apiPort) throws IOException {
private String makeConfigFile(
int port,
int apiPort
) throws IOException {
return makeConfigFile(port, apiPort, null);
}
private String makeConfigFile(
int port,
int apiPort,
SwapJobConfig swapCfg
) throws IOException {
File wlgb = folder.newFolder();
File config = folder.newFile();
PrintWriter writer = new PrintWriter(config);
writer.println("{\n" +
"\t\"port\": " + port + ",\n" +
"\t\"rootGitDirectory\": \"" + wlgb.getAbsolutePath() + "\",\n" +
"\t\"apiBaseUrl\": \"http://127.0.0.1:" + apiPort + "/api/v0\",\n" +
"\t\"username\": \"\",\n" +
"\t\"password\": \"\",\n" +
"\t\"postbackBaseUrl\": \"http://127.0.0.1:" + port + "\",\n" +
"\t\"serviceName\": \"Overleaf\"\n," +
" \"oauth2\": {\n" +
" \"oauth2ClientID\": \"clientID\",\n" +
" \"oauth2ClientSecret\": \"oauth2 client secret\",\n" +
" \"oauth2Server\": \"https://www.overleaf.com\"\n" +
" }\n" +
"}\n");
String cfgStr =
"{\n" +
" \"port\": " + port + ",\n" +
" \"rootGitDirectory\": \"" +
wlgb.getAbsolutePath() +
"\",\n" +
" \"apiBaseUrl\": \"http://127.0.0.1:" +
apiPort +
"/api/v0\",\n" +
" \"username\": \"\",\n" +
" \"password\": \"\",\n" +
" \"postbackBaseUrl\": \"http://127.0.0.1:" +
port +
"\",\n" +
" \"serviceName\": \"Overleaf\",\n" +
" \"oauth2\": {\n" +
" \"oauth2ClientID\": \"clientID\",\n" +
" \"oauth2ClientSecret\": \"oauth2 client secret\",\n" +
" \"oauth2Server\": \"https://www.overleaf.com\"\n" +
" }";
if (swapCfg != null) {
cfgStr += ",\n" +
" \"swapStore\": {\n" +
" \"type\": \"memory\"\n" +
" },\n" +
" \"swapJob\": {\n" +
" \"minProjects\": " +
swapCfg.getMinProjects() +
",\n" +
" \"lowGiB\": " +
swapCfg.getLowGiB() +
",\n" +
" \"highGiB\": " +
swapCfg.getHighGiB() +
",\n" +
" \"intervalMillis\": " +
swapCfg.getIntervalMillis() +
"\n" +
" }\n";
}
cfgStr += "}\n";
writer.print(cfgStr);
writer.close();
return config.getAbsolutePath();
}
private Path getResource(String path) {
return Paths.get("src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest" + path);
return Paths.get(
"src/test/resources/" +
"uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest" + path
);
}
private InputStream getResourceAsStream(String path) {

View file

@ -3,15 +3,25 @@ package uk.ac.ic.wlgitbridge.bridge;
import org.junit.Before;
import org.junit.Test;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import java.io.IOException;
import java.util.ArrayDeque;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Created by winston on 20/08/2016.
@ -56,4 +66,21 @@ public class BridgeTest {
verify(swapJob).stop();
}
@Test
public void updatingRepositorySetsLastAccessedTime(
) throws IOException, GitUserException {
ProjectRepo repo = mock(ProjectRepo.class);
when(repo.getProjectName()).thenReturn("asdf");
when(dbStore.getProjectState("asdf")).thenReturn(ProjectState.PRESENT);
when(
snapshotAPI.getSnapshotsForProjectAfterVersion(
any(),
any(),
anyInt()
)
).thenReturn(new ArrayDeque<Snapshot>());
bridge.updateRepository(null, repo);
verify(dbStore).setLastAccessedTime(eq("asdf"), any());
}
}

View file

@ -0,0 +1,90 @@
[
{
"project": "testproj1",
"getDoc": {
"versionID": 1,
"createdAt": "2014-11-30T18:40:58.123Z",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1"
},
"getSavedVers": [
{
"versionID": 1,
"comment": "added more info on doc GET and error details",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1",
"createdAt": "2014-11-30T18:47:01.456Z"
}
],
"getForVers": [
{
"versionID": 1,
"srcs": [
{
"content": "content\n",
"path": "main.tex"
},
{
"content": "This text is from another file.",
"path": "foo/bar/test.tex"
}
],
"atts": [
{
"url": "http://127.0.0.1:3874/state/testproj1/overleaf-white-410.png",
"path": "overleaf-white-410.png"
}
]
}
],
"push": "success",
"postback": {
"type": "success",
"versionID": 2
}
},
{
"project": "testproj2",
"getDoc": {
"versionID": 1,
"createdAt": "2014-11-30T18:40:58.123Z",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1"
},
"getSavedVers": [
{
"versionID": 1,
"comment": "added more info on doc GET and error details",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1",
"createdAt": "2014-11-30T18:47:01.456Z"
}
],
"getForVers": [
{
"versionID": 1,
"srcs": [
{
"content": "different content\n",
"path": "main.tex"
},
{
"content": "a different one",
"path": "foo/bar/test.tex"
}
],
"atts": [
{
"url": "http://127.0.0.1:3874/state/testproj2/editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png",
"path": "editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png"
}
]
}
],
"push": "success",
"postback": {
"type": "success",
"versionID": 2
}
}
]