mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-29 07:03:37 -05:00
Implement and test the swap job, and add integration test
This commit is contained in:
parent
dd5694104d
commit
9936fbe3c9
20 changed files with 258 additions and 45 deletions
|
@ -25,7 +25,7 @@ public class GitBridgeApp implements Runnable {
|
|||
"usage: writelatex-git-bridge config_file";
|
||||
|
||||
private String configFilePath;
|
||||
private Config config;
|
||||
Config config;
|
||||
private GitBridgeServer server;
|
||||
|
||||
/**
|
||||
|
|
|
@ -98,6 +98,14 @@ public class Config implements JSONSource {
|
|||
postbackURL += "/";
|
||||
}
|
||||
oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
|
||||
swapStore = new Gson().fromJson(
|
||||
configObject.get("swapStore"),
|
||||
SwapStoreConfig.class
|
||||
);
|
||||
swapJob = new Gson().fromJson(
|
||||
configObject.get("swapJob"),
|
||||
SwapJobConfig.class
|
||||
);
|
||||
}
|
||||
|
||||
public String getSanitisedString() {
|
||||
|
|
|
@ -10,6 +10,9 @@ import java.util.Collection;
|
|||
*/
|
||||
public interface RepoStore {
|
||||
|
||||
/* Still need to get rid of these two methods.
|
||||
Main dependency: GitRepoStore needs a Repository which needs a directory.
|
||||
Instead, use a visitor or something. */
|
||||
String getRepoStorePath();
|
||||
|
||||
File getRootDirectory();
|
||||
|
@ -20,7 +23,7 @@ public interface RepoStore {
|
|||
|
||||
long totalSize();
|
||||
|
||||
/*
|
||||
/**
|
||||
* Tars and bzip2s the .git directory of the given project. Throws an
|
||||
* IOException if the project doesn't exist. The returned stream is a copy
|
||||
* of the original .git directory, which must be deleted using remove().
|
||||
|
|
|
@ -5,9 +5,6 @@ package uk.ac.ic.wlgitbridge.bridge.swap.job;
|
|||
*/
|
||||
public class SwapJobConfig {
|
||||
|
||||
public static final SwapJobConfig DEFAULT =
|
||||
new SwapJobConfig(1, 1, 2, 3600000);
|
||||
|
||||
private final int minProjects;
|
||||
private final int lowGiB;
|
||||
private final int highGiB;
|
||||
|
|
|
@ -19,6 +19,10 @@ public class InMemorySwapStore implements SwapStore {
|
|||
store = new HashMap<>();
|
||||
}
|
||||
|
||||
public InMemorySwapStore(SwapStoreConfig __) {
|
||||
this();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void upload(
|
||||
String projectName,
|
||||
|
|
|
@ -17,6 +17,7 @@ public interface SwapStore {
|
|||
|
||||
{
|
||||
put("noop", NoopSwapStore::new);
|
||||
put("memory", InMemorySwapStore::new);
|
||||
put("s3", S3SwapStore::new);
|
||||
}
|
||||
|
||||
|
|
|
@ -19,10 +19,10 @@ import javax.servlet.http.HttpServletRequest;
|
|||
/* */
|
||||
public class WLReceivePackFactory implements ReceivePackFactory<HttpServletRequest> {
|
||||
|
||||
private final Bridge bridgeAPI;
|
||||
private final Bridge bridge;
|
||||
|
||||
public WLReceivePackFactory(Bridge bridgeAPI) {
|
||||
this.bridgeAPI = bridgeAPI;
|
||||
public WLReceivePackFactory(Bridge bridge) {
|
||||
this.bridge = bridge;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -33,7 +33,7 @@ public class WLReceivePackFactory implements ReceivePackFactory<HttpServletReque
|
|||
if (hostname == null) {
|
||||
hostname = httpServletRequest.getLocalName();
|
||||
}
|
||||
receivePack.setPreReceiveHook(new WriteLatexPutHook(bridgeAPI, hostname, oauth2));
|
||||
receivePack.setPreReceiveHook(new WriteLatexPutHook(bridge, hostname, oauth2));
|
||||
return receivePack;
|
||||
}
|
||||
|
||||
|
|
|
@ -26,12 +26,12 @@ import java.util.Iterator;
|
|||
*/
|
||||
public class WriteLatexPutHook implements PreReceiveHook {
|
||||
|
||||
private final Bridge bridgeAPI;
|
||||
private final Bridge bridge;
|
||||
private final String hostname;
|
||||
private final Credential oauth2;
|
||||
|
||||
public WriteLatexPutHook(Bridge bridgeAPI, String hostname, Credential oauth2) {
|
||||
this.bridgeAPI = bridgeAPI;
|
||||
public WriteLatexPutHook(Bridge bridge, String hostname, Credential oauth2) {
|
||||
this.bridge = bridge;
|
||||
this.hostname = hostname;
|
||||
this.oauth2 = oauth2;
|
||||
}
|
||||
|
@ -75,7 +75,7 @@ public class WriteLatexPutHook implements PreReceiveHook {
|
|||
private void handleReceiveCommand(Credential oauth2, Repository repository, ReceiveCommand receiveCommand) throws IOException, GitUserException {
|
||||
checkBranch(receiveCommand);
|
||||
checkForcedPush(receiveCommand);
|
||||
bridgeAPI.putDirectoryContentsToProjectWithName(
|
||||
bridge.putDirectoryContentsToProjectWithName(
|
||||
oauth2,
|
||||
repository.getWorkTree().getName(),
|
||||
getPushedDirectoryContents(repository,
|
||||
|
|
|
@ -37,7 +37,7 @@ import java.util.EnumSet;
|
|||
*/
|
||||
public class GitBridgeServer {
|
||||
|
||||
private final Bridge bridgeAPI;
|
||||
private final Bridge bridge;
|
||||
|
||||
private final Server jettyServer;
|
||||
|
||||
|
@ -58,7 +58,7 @@ public class GitBridgeServer {
|
|||
).resolve(".wlgb").resolve("wlgb.db").toFile()
|
||||
);
|
||||
SwapStore swapStore = SwapStore.fromConfig(config.getSwapStore());
|
||||
bridgeAPI = Bridge.make(
|
||||
bridge = Bridge.make(
|
||||
repoStore,
|
||||
dbStore,
|
||||
swapStore,
|
||||
|
@ -83,6 +83,7 @@ public class GitBridgeServer {
|
|||
public void start() {
|
||||
try {
|
||||
jettyServer.start();
|
||||
bridge.startSwapJob();
|
||||
Log.info(Util.getServiceName() + "-Git Bridge server started");
|
||||
Log.info("Listening on port: " + port);
|
||||
Log.info("Bridged to: " + apiBaseURL);
|
||||
|
@ -118,7 +119,7 @@ public class GitBridgeServer {
|
|||
|
||||
HandlerCollection handlers = new HandlerList();
|
||||
handlers.addHandler(initResourceHandler());
|
||||
handlers.addHandler(new PostbackHandler(bridgeAPI));
|
||||
handlers.addHandler(new PostbackHandler(bridge));
|
||||
handlers.addHandler(new DefaultHandler());
|
||||
|
||||
api.setHandler(handlers);
|
||||
|
@ -143,7 +144,7 @@ public class GitBridgeServer {
|
|||
new ServletHolder(
|
||||
new WLGitServlet(
|
||||
servletContextHandler,
|
||||
bridgeAPI
|
||||
bridge
|
||||
)
|
||||
),
|
||||
"/*"
|
||||
|
@ -152,7 +153,7 @@ public class GitBridgeServer {
|
|||
}
|
||||
|
||||
private Handler initResourceHandler() {
|
||||
ResourceHandler resourceHandler = new FileHandler(bridgeAPI);
|
||||
ResourceHandler resourceHandler = new FileHandler(bridge);
|
||||
resourceHandler.setResourceBase(
|
||||
new File(rootGitDirectoryPath, ".wlgb/atts").getAbsolutePath()
|
||||
);
|
||||
|
|
|
@ -17,7 +17,7 @@ public class PostbackContents implements JSONSource {
|
|||
|
||||
private static final String CODE_SUCCESS = "upToDate";
|
||||
|
||||
private final Bridge bridgeAPI;
|
||||
private final Bridge bridge;
|
||||
private final String projectName;
|
||||
private final String postbackKey;
|
||||
|
||||
|
@ -26,8 +26,8 @@ public class PostbackContents implements JSONSource {
|
|||
private int versionID;
|
||||
private SnapshotPostException exception;
|
||||
|
||||
public PostbackContents(Bridge bridgeAPI, String projectName, String postbackKey, String contents) {
|
||||
this.bridgeAPI = bridgeAPI;
|
||||
public PostbackContents(Bridge bridge, String projectName, String postbackKey, String contents) {
|
||||
this.bridge = bridge;
|
||||
this.projectName = projectName;
|
||||
this.postbackKey = postbackKey;
|
||||
snapshotPostExceptionBuilder = new SnapshotPostExceptionBuilder();
|
||||
|
@ -43,9 +43,9 @@ public class PostbackContents implements JSONSource {
|
|||
|
||||
public void processPostback() throws UnexpectedPostbackException {
|
||||
if (exception == null) {
|
||||
bridgeAPI.postbackReceivedSuccessfully(projectName, postbackKey, versionID);
|
||||
bridge.postbackReceivedSuccessfully(projectName, postbackKey, versionID);
|
||||
} else {
|
||||
bridgeAPI.postbackReceivedWithException(projectName, postbackKey, exception);
|
||||
bridge.postbackReceivedWithException(projectName, postbackKey, exception);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -19,10 +19,10 @@ import java.io.IOException;
|
|||
*/
|
||||
public class PostbackHandler extends AbstractHandler {
|
||||
|
||||
private final Bridge bridgeAPI;
|
||||
private final Bridge bridge;
|
||||
|
||||
public PostbackHandler(Bridge bridgeAPI) {
|
||||
this.bridgeAPI = bridgeAPI;
|
||||
public PostbackHandler(Bridge bridge) {
|
||||
this.bridge = bridge;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -39,7 +39,7 @@ public class PostbackHandler extends AbstractHandler {
|
|||
String projectName = parts[1];
|
||||
String postbackKey = parts[2];
|
||||
Log.info(baseRequest.getMethod() + " <- " + baseRequest.getHttpURI());
|
||||
PostbackContents postbackContents = new PostbackContents(bridgeAPI, projectName, postbackKey, contents);
|
||||
PostbackContents postbackContents = new PostbackContents(bridge, projectName, postbackKey, contents);
|
||||
JsonObject body = new JsonObject();
|
||||
|
||||
try {
|
||||
|
|
|
@ -1,13 +1,14 @@
|
|||
package uk.ac.ic.wlgitbridge;
|
||||
package uk.ac.ic.wlgitbridge.application;
|
||||
|
||||
import com.ning.http.client.AsyncHttpClient;
|
||||
import com.ning.http.client.Response;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.eclipse.jgit.api.errors.GitAPIException;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
import uk.ac.ic.wlgitbridge.application.GitBridgeApp;
|
||||
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.servermock.server.MockSnapshotServer;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.servermock.state.SnapshotAPIState;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.servermock.state.SnapshotAPIStateBuilder;
|
||||
|
@ -102,6 +103,9 @@ public class WLGitBridgeIntegrationTest {
|
|||
put("canServePushedFiles", new HashMap<String, SnapshotAPIState>() {{
|
||||
put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/canServePushedFiles/state/state.json")).build());
|
||||
}});
|
||||
put("wlgbCanSwapProjects", new HashMap<String, SnapshotAPIState>() {{
|
||||
put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/wlgbCanSwapProjects/state/state.json")).build());
|
||||
}});
|
||||
}};
|
||||
|
||||
@Rule
|
||||
|
@ -587,6 +591,38 @@ public class WLGitBridgeIntegrationTest {
|
|||
wlgb.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void wlgbCanSwapProjects(
|
||||
) throws IOException, GitAPIException, InterruptedException {
|
||||
MockSnapshotServer server = new MockSnapshotServer(
|
||||
3874,
|
||||
getResource("/wlgbCanSwapProjects").toFile()
|
||||
);
|
||||
server.start();
|
||||
server.setState(states.get("wlgbCanSwapProjects").get("state"));
|
||||
GitBridgeApp wlgb = new GitBridgeApp(new String[] {
|
||||
makeConfigFile(33874, 3874, new SwapJobConfig(1, 0, 0, 250))
|
||||
});
|
||||
wlgb.run();
|
||||
File dir = folder.newFolder();
|
||||
File rootGitDir = new File(wlgb.config.getRootGitDirectory());
|
||||
File testProj1ServerDir = new File(rootGitDir, "testproj1");
|
||||
File testProj2ServerDir = new File(rootGitDir, "testproj2");
|
||||
File testProj1Dir = cloneRepository("testproj1", 33874, dir);
|
||||
assertTrue(testProj1ServerDir.exists());
|
||||
assertFalse(testProj2ServerDir.exists());
|
||||
cloneRepository("testproj2", 33874, dir);
|
||||
while (testProj1ServerDir.exists());
|
||||
assertFalse(testProj1ServerDir.exists());
|
||||
assertTrue(testProj2ServerDir.exists());
|
||||
FileUtils.deleteDirectory(testProj1Dir);
|
||||
cloneRepository("testproj1", 33874, dir);
|
||||
while (testProj2ServerDir.exists());
|
||||
assertTrue(testProj1ServerDir.exists());
|
||||
assertFalse(testProj2ServerDir.exists());
|
||||
wlgb.stop();
|
||||
}
|
||||
|
||||
private File cloneRepository(String repositoryName, int port, File dir) throws IOException, InterruptedException {
|
||||
String repo = "git clone http://127.0.0.1:" + port + "/" + repositoryName + ".git";
|
||||
Process gitProcess = runtime.exec(repo, null, dir);
|
||||
|
@ -606,30 +642,72 @@ public class WLGitBridgeIntegrationTest {
|
|||
return repositoryDir;
|
||||
}
|
||||
|
||||
private String makeConfigFile(int port, int apiPort) throws IOException {
|
||||
private String makeConfigFile(
|
||||
int port,
|
||||
int apiPort
|
||||
) throws IOException {
|
||||
return makeConfigFile(port, apiPort, null);
|
||||
}
|
||||
|
||||
private String makeConfigFile(
|
||||
int port,
|
||||
int apiPort,
|
||||
SwapJobConfig swapCfg
|
||||
) throws IOException {
|
||||
File wlgb = folder.newFolder();
|
||||
File config = folder.newFile();
|
||||
PrintWriter writer = new PrintWriter(config);
|
||||
writer.println("{\n" +
|
||||
"\t\"port\": " + port + ",\n" +
|
||||
"\t\"rootGitDirectory\": \"" + wlgb.getAbsolutePath() + "\",\n" +
|
||||
"\t\"apiBaseUrl\": \"http://127.0.0.1:" + apiPort + "/api/v0\",\n" +
|
||||
"\t\"username\": \"\",\n" +
|
||||
"\t\"password\": \"\",\n" +
|
||||
"\t\"postbackBaseUrl\": \"http://127.0.0.1:" + port + "\",\n" +
|
||||
"\t\"serviceName\": \"Overleaf\"\n," +
|
||||
String cfgStr =
|
||||
"{\n" +
|
||||
" \"port\": " + port + ",\n" +
|
||||
" \"rootGitDirectory\": \"" +
|
||||
wlgb.getAbsolutePath() +
|
||||
"\",\n" +
|
||||
" \"apiBaseUrl\": \"http://127.0.0.1:" +
|
||||
apiPort +
|
||||
"/api/v0\",\n" +
|
||||
" \"username\": \"\",\n" +
|
||||
" \"password\": \"\",\n" +
|
||||
" \"postbackBaseUrl\": \"http://127.0.0.1:" +
|
||||
port +
|
||||
"\",\n" +
|
||||
" \"serviceName\": \"Overleaf\",\n" +
|
||||
" \"oauth2\": {\n" +
|
||||
" \"oauth2ClientID\": \"clientID\",\n" +
|
||||
" \"oauth2ClientSecret\": \"oauth2 client secret\",\n" +
|
||||
" \"oauth2Server\": \"https://www.overleaf.com\"\n" +
|
||||
" }\n" +
|
||||
"}\n");
|
||||
" }";
|
||||
if (swapCfg != null) {
|
||||
cfgStr += ",\n" +
|
||||
" \"swapStore\": {\n" +
|
||||
" \"type\": \"memory\"\n" +
|
||||
" },\n" +
|
||||
" \"swapJob\": {\n" +
|
||||
" \"minProjects\": " +
|
||||
swapCfg.getMinProjects() +
|
||||
",\n" +
|
||||
" \"lowGiB\": " +
|
||||
swapCfg.getLowGiB() +
|
||||
",\n" +
|
||||
" \"highGiB\": " +
|
||||
swapCfg.getHighGiB() +
|
||||
",\n" +
|
||||
" \"intervalMillis\": " +
|
||||
swapCfg.getIntervalMillis() +
|
||||
"\n" +
|
||||
" }\n";
|
||||
}
|
||||
cfgStr += "}\n";
|
||||
writer.print(cfgStr);
|
||||
writer.close();
|
||||
return config.getAbsolutePath();
|
||||
}
|
||||
|
||||
private Path getResource(String path) {
|
||||
return Paths.get("src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest" + path);
|
||||
return Paths.get(
|
||||
"src/test/resources/" +
|
||||
"uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest" + path
|
||||
);
|
||||
}
|
||||
|
||||
private InputStream getResourceAsStream(String path) {
|
|
@ -3,15 +3,25 @@ package uk.ac.ic.wlgitbridge.bridge;
|
|||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
|
||||
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
|
||||
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
|
||||
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
|
||||
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
|
||||
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
|
||||
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
|
||||
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayDeque;
|
||||
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Matchers.anyInt;
|
||||
import static org.mockito.Matchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Created by winston on 20/08/2016.
|
||||
|
@ -56,4 +66,21 @@ public class BridgeTest {
|
|||
verify(swapJob).stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void updatingRepositorySetsLastAccessedTime(
|
||||
) throws IOException, GitUserException {
|
||||
ProjectRepo repo = mock(ProjectRepo.class);
|
||||
when(repo.getProjectName()).thenReturn("asdf");
|
||||
when(dbStore.getProjectState("asdf")).thenReturn(ProjectState.PRESENT);
|
||||
when(
|
||||
snapshotAPI.getSnapshotsForProjectAfterVersion(
|
||||
any(),
|
||||
any(),
|
||||
anyInt()
|
||||
)
|
||||
).thenReturn(new ArrayDeque<Snapshot>());
|
||||
bridge.updateRepository(null, repo);
|
||||
verify(dbStore).setLastAccessedTime(eq("asdf"), any());
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,90 @@
|
|||
[
|
||||
{
|
||||
"project": "testproj1",
|
||||
"getDoc": {
|
||||
"versionID": 1,
|
||||
"createdAt": "2014-11-30T18:40:58.123Z",
|
||||
"email": "jdleesmiller+1@gmail.com",
|
||||
"name": "John+1"
|
||||
},
|
||||
"getSavedVers": [
|
||||
{
|
||||
"versionID": 1,
|
||||
"comment": "added more info on doc GET and error details",
|
||||
"email": "jdleesmiller+1@gmail.com",
|
||||
"name": "John+1",
|
||||
"createdAt": "2014-11-30T18:47:01.456Z"
|
||||
}
|
||||
],
|
||||
"getForVers": [
|
||||
{
|
||||
"versionID": 1,
|
||||
"srcs": [
|
||||
{
|
||||
"content": "content\n",
|
||||
"path": "main.tex"
|
||||
},
|
||||
{
|
||||
"content": "This text is from another file.",
|
||||
"path": "foo/bar/test.tex"
|
||||
}
|
||||
],
|
||||
"atts": [
|
||||
{
|
||||
"url": "http://127.0.0.1:3874/state/testproj1/overleaf-white-410.png",
|
||||
"path": "overleaf-white-410.png"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"push": "success",
|
||||
"postback": {
|
||||
"type": "success",
|
||||
"versionID": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"project": "testproj2",
|
||||
"getDoc": {
|
||||
"versionID": 1,
|
||||
"createdAt": "2014-11-30T18:40:58.123Z",
|
||||
"email": "jdleesmiller+1@gmail.com",
|
||||
"name": "John+1"
|
||||
},
|
||||
"getSavedVers": [
|
||||
{
|
||||
"versionID": 1,
|
||||
"comment": "added more info on doc GET and error details",
|
||||
"email": "jdleesmiller+1@gmail.com",
|
||||
"name": "John+1",
|
||||
"createdAt": "2014-11-30T18:47:01.456Z"
|
||||
}
|
||||
],
|
||||
"getForVers": [
|
||||
{
|
||||
"versionID": 1,
|
||||
"srcs": [
|
||||
{
|
||||
"content": "different content\n",
|
||||
"path": "main.tex"
|
||||
},
|
||||
{
|
||||
"content": "a different one",
|
||||
"path": "foo/bar/test.tex"
|
||||
}
|
||||
],
|
||||
"atts": [
|
||||
{
|
||||
"url": "http://127.0.0.1:3874/state/testproj2/editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png",
|
||||
"path": "editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"push": "success",
|
||||
"postback": {
|
||||
"type": "success",
|
||||
"versionID": 2
|
||||
}
|
||||
}
|
||||
]
|
|
@ -0,0 +1 @@
|
|||
This text is from another file.
|
|
@ -0,0 +1 @@
|
|||
content
|
Binary file not shown.
After Width: | Height: | Size: 8.6 KiB |
Binary file not shown.
After Width: | Height: | Size: 3.5 KiB |
|
@ -0,0 +1 @@
|
|||
a different one
|
|
@ -0,0 +1 @@
|
|||
different content
|
Loading…
Reference in a new issue