github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-04-07 14:32:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #45 from overleaf/sk-migrate-repos-v1-v2

Browse source 
Migrate repos from v1 to v2
This commit is contained in:
Shane Kilkelly 2018-12-06 10:30:07 +00:00 • committed by GitHub
commit 002d862389
20 changed files with 323 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
services/git-bridge/pom.xml
View file

@ -9,6 +9,7 @@
<version>1.0-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!--<maven.test.skip>true</maven.test.skip>-->
</properties>
<build>
<plugins>
@ -22,6 +23,15 @@
<compilerArgument></compilerArgument>
</configuration>
</plugin>
<!-- Workaround, test loader crashes without this configuration option -->
<!-- See: https://stackoverflow.com/questions/53010200/maven-surefire-could-not-find-forkedbooter-class -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>-Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
</configuration>
</plugin>
<!-- https://mvnrepository.com/artifact/org.apache.maven.plugins/maven-assembly-plugin -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>

52
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java
View file

@ -37,6 +37,7 @@ import uk.ac.ic.wlgitbridge.server.PostbackContents;
import uk.ac.ic.wlgitbridge.server.PostbackHandler;
import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
import uk.ac.ic.wlgitbridge.snapshot.push.PostbackManager;
import uk.ac.ic.wlgitbridge.snapshot.push.PostbackPromise;
@ -299,7 +300,7 @@ public class Bridge {
* Synchronises the given repository with Overleaf.
*
* It acquires the project lock and calls
* {@link #getUpdatedRepoCritical(Optional, String)}.
* {@link #getUpdatedRepoCritical(Optional, String, GetDocResult)}.
* @param oauth2 The oauth2 to use
* @param projectName The name of the project
* @throws IOException
@ -310,11 +311,13 @@ public class Bridge {
String projectName
) throws IOException, GitUserException {
try (LockGuard __ = lock.lockGuard(projectName)) {
if (!snapshotAPI.projectExists(oauth2, projectName)) {
Optional<GetDocResult> maybeDoc = snapshotAPI.getDoc(oauth2, projectName);
if (!maybeDoc.isPresent()) {
throw new RepositoryNotFoundException(projectName);
}
GetDocResult doc = maybeDoc.get();
Log.info("[{}] Updating repository", projectName);
return getUpdatedRepoCritical(oauth2, projectName);
return getUpdatedRepoCritical(oauth2, projectName, doc);
}
}
@ -346,14 +349,51 @@ public class Bridge {
*/
private ProjectRepo getUpdatedRepoCritical(
Optional<Credential> oauth2,
String projectName
String projectName,
GetDocResult doc
) throws IOException, GitUserException {
ProjectRepo repo;
ProjectState state = dbStore.getProjectState(projectName);
switch (state) {
case NOT_PRESENT:
repo = repoStore.initRepo(projectName);
break;
Log.info("[{}] Repo not present", projectName);
String migratedFromID = doc.getMigratedFromID();
if (migratedFromID != null) {
Log.info("[{}] Has a migratedFromId: {}", projectName, migratedFromID);
try (LockGuard __ = lock.lockGuard(migratedFromID)) {
ProjectState sourceState = dbStore.getProjectState(migratedFromID);
switch (sourceState) {
case NOT_PRESENT:
// Normal init-repo
Log.info("[{}] migrated-from project not present, proceed as normal",
projectName
);
repo = repoStore.initRepo(projectName);
break;
case SWAPPED:
// Swap back and then copy
swapJob.restore(migratedFromID);
/* Fallthrough */
default:
// Copy data, and set version to zero
Log.info("[{}] Init from other project: {}",
projectName,
migratedFromID
);
repo = repoStore.initRepoFromExisting(projectName, migratedFromID);
dbStore.setLatestVersionForProject(migratedFromID, 0);
dbStore.setLastAccessedTime(
migratedFromID,
Timestamp.valueOf(LocalDateTime.now())
);
}
}
break;
} else {
repo = repoStore.initRepo(projectName);
break;
}
case SWAPPED:
swapJob.restore(projectName);
/* Fallthrough */

22
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java
View file

@ -4,6 +4,7 @@ import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Project;
import uk.ac.ic.wlgitbridge.util.Tar;
@ -74,6 +75,27 @@ public class FSGitRepoStore implements RepoStore {
ret, Optional.of(maxFileSize), Optional.empty());
}
@Override
public ProjectRepo initRepoFromExisting(
String project, String fromProject
) throws IOException {
String repoRoot = getRepoStorePath();
String sourcePath = repoRoot + "/" + fromProject;
String destinationPath = repoRoot + "/" + project;
Log.info("[{}] Init repo by copying data from: {}, to: {}",
project,
sourcePath,
destinationPath
);
File source = new File(sourcePath);
File destination = new File(destinationPath);
FileUtils.copyDirectory(source, destination);
GitProjectRepo ret = GitProjectRepo.fromName(project);
ret.useExistingRepository(this);
return new WalkOverrideGitRepo(
ret, Optional.of(maxFileSize), Optional.empty());
}
@Override
public ProjectRepo getExistingRepo(String project) throws IOException {
GitProjectRepo ret = GitProjectRepo.fromName(project);

2
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java
View file

@ -22,6 +22,8 @@ public interface RepoStore {
ProjectRepo initRepo(String project) throws IOException;
ProjectRepo initRepoFromExisting(String project, String fromProject) throws IOException;
ProjectRepo getExistingRepo(String project) throws IOException;
ProjectRepo useJGitRepo(Repository repo, ObjectId commitId);

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApiFacade.java
View file

@ -44,6 +44,20 @@ public class SnapshotApiFacade {
}
}
public Optional<GetDocResult> getDoc(
Optional<Credential> oauth2,
String projectName
) throws FailedConnectionException, GitUserException {
try {
GetDocResult doc = SnapshotApi
.getResult(api.getDoc(oauth2, projectName));
doc.getVersionID();
return Optional.of(doc);
} catch (InvalidProjectException e) {
return Optional.empty();
}
}
public Deque<Snapshot> getSnapshots(
Optional<Credential> oauth2,
String projectName,

30
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/base/MissingRepositoryException.java
View file

@ -15,11 +15,31 @@ public class MissingRepositoryException extends SnapshotAPIException {
"If this problem persists, please contact us."
);
public static final List<String> EXPORTED_TO_V2 = Arrays.asList(
"This Overleaf project has been moved to Overleaf v2, and git access is temporarily unsupported.",
"",
"See https://www.overleaf.com/help/342 for more information."
);
static List<String> buildExportedToV2Message(String remoteUrl) {
if (remoteUrl == null) {
return Arrays.asList(
"This Overleaf project has been moved to Overleaf v2 and cannot be used with git at this time.",
"",
"If this error persists, please contact us at support@overleaf.com, or",
"see https://www.overleaf.com/help/342 for more information."
);
} else {
return Arrays.asList(
"This Overleaf project has been moved to Overleaf v2 and has a new identifier.",
"Please update your remote to:",
"",
" " + remoteUrl,
"",
"Assuming you are using the default \"origin\" remote, the following commands",
"will change the remote for you:",
"",
" git remote set-url origin " + remoteUrl,
"",
"If this does not work, please contact us at support@overleaf.com, or",
"see https://www.overleaf.com/help/342 for more information."
);
}
}
private List<String> descriptionLines;

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/base/Request.java
View file

@ -79,9 +79,17 @@ public abstract class Request<T extends Result> {
try {
JsonObject json = Instance.gson.fromJson(httpCause.getContent(), JsonObject.class);
String message = json.get("message").getAsString();
String newRemote;
if (json.has("newRemote")) {
newRemote = json.get("newRemote").getAsString();
} else {
newRemote = null;
}
if ("Exported to v2".equals(message)) {
throw new MissingRepositoryException(MissingRepositoryException.EXPORTED_TO_V2);
throw new MissingRepositoryException(
MissingRepositoryException.buildExportedToV2Message(newRemote)
);
}
} catch (IllegalStateException
| ClassCastException

15
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getdoc/GetDocResult.java
View file

@ -19,6 +19,7 @@ public class GetDocResult extends Result {
private int error;
private int versionID;
private String migratedFromID;
private String createdAt;
private WLUser user;
@ -37,7 +38,8 @@ public class GetDocResult extends Result {
int versionID,
String createdAt,
String email,
String name
String name,
String migratedFromID
) {
if (error == null) {
this.error = -1;
@ -47,6 +49,7 @@ public class GetDocResult extends Result {
this.versionID = versionID;
this.createdAt = createdAt;
this.user = new WLUser(name, email);
this.migratedFromID = migratedFromID;
}
@Override
@ -59,6 +62,9 @@ public class GetDocResult extends Result {
latestVerBy.addProperty("email", getEmail());
latestVerBy.addProperty("name", getName());
jsonThis.add("latestVerBy", latestVerBy);
if (migratedFromID != null) {
jsonThis.addProperty("migratedFromId", migratedFromID);
}
} else {
jsonThis.addProperty("status", error);
String message;
@ -93,6 +99,11 @@ public class GetDocResult extends Result {
} else {
versionID = jsonObject.get("latestVerId").getAsInt();
createdAt = jsonObject.get("latestVerAt").getAsString();
if (jsonObject.has("migratedFromId")) {
migratedFromID = jsonObject.get("migratedFromId").getAsString();
} else {
migratedFromID = null;
}
String name = null;
String email = null;
JsonElement latestVerBy = jsonObject.get("latestVerBy");
@ -126,4 +137,6 @@ public class GetDocResult extends Result {
return user.getEmail();
}
public String getMigratedFromID() { return migratedFromID; }
}

3
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/servermock/state/SnapshotAPIState.java
View file

@ -48,7 +48,8 @@ public class SnapshotAPIState {
243,
"2014-11-30T18:40:58Z",
"jdleesmiller+1@gmail.com",
"John+1"
"John+1",
null
)
);

7
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/servermock/state/SnapshotAPIStateBuilder.java
View file

@ -83,6 +83,10 @@ public class SnapshotAPIStateBuilder {
String projectName,
JsonObject jsonGetDoc
) {
String migratedFromId = null;
if (jsonGetDoc.has("migratedFromId")) {
migratedFromId = jsonGetDoc.get("migratedFromId").getAsString();
}
getDoc.put(
projectName,
new GetDocResult(
@ -90,7 +94,8 @@ public class SnapshotAPIStateBuilder {
jsonGetDoc.get("versionID").getAsInt(),
jsonGetDoc.get("createdAt").getAsString(),
jsonGetDoc.get("email").getAsString(),
jsonGetDoc.get("name").getAsString()
jsonGetDoc.get("name").getAsString(),
migratedFromId
)
);
}

43
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java
View file

@ -117,6 +117,12 @@ public class WLGitBridgeIntegrationTest {
put("pushSubmoduleFailsWithInvalidGitRepo", new HashMap<String, SnapshotAPIState>() {{
put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/pushSubmoduleFailsWithInvalidGitRepo/state/state.json")).build());
}});
put("canMigrateRepository", new HashMap<String, SnapshotAPIState>() {{
put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/canMigrateRepository/state/state.json")).build());
}});
put("skipMigrationWhenMigratedFromMissing", new HashMap<String, SnapshotAPIState>() {{
put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/skipMigrationWhenMigratedFromMissing/state/state.json")).build());
}});
}};
@Rule
@ -787,6 +793,43 @@ public class WLGitBridgeIntegrationTest {
assertNotEquals(0, gitProcess.waitFor());
}
@Test
public void canMigrateRepository() throws IOException, GitAPIException, InterruptedException {
int gitBridgePort = 33881;
int mockServerPort = 3881;
MockSnapshotServer server = new MockSnapshotServer(mockServerPort, getResource("/canMigrateRepository").toFile());
server.start();
server.setState(states.get("canMigrateRepository").get("state"));
GitBridgeApp wlgb = new GitBridgeApp(new String[] {
makeConfigFile(gitBridgePort, mockServerPort)
});
wlgb.run();
File testprojDir = gitClone("testproj", gitBridgePort, dir);
File testprojDir2 = gitClone("testproj2", gitBridgePort, dir);
wlgb.stop();
// Second project content is equal to content of the first
assertTrue(FileUtil.gitDirectoriesAreEqual(getResource("/canMigrateRepository/state/testproj"), testprojDir2.toPath()));
}
@Test
public void skipMigrationWhenMigratedFromMissing() throws IOException, GitAPIException, InterruptedException {
int gitBridgePort = 33882;
int mockServerPort = 3882;
MockSnapshotServer server = new MockSnapshotServer(mockServerPort, getResource("/skipMigrationWhenMigratedFromMissing").toFile());
server.start();
server.setState(states.get("skipMigrationWhenMigratedFromMissing").get("state"));
GitBridgeApp wlgb = new GitBridgeApp(new String[] {
makeConfigFile(gitBridgePort, mockServerPort)
});
wlgb.run();
// don't clone the source project first
File testprojDir2 = gitClone("testproj2", gitBridgePort, dir);
wlgb.stop();
assertTrue(FileUtil.gitDirectoriesAreEqual(getResource("/skipMigrationWhenMigratedFromMissing/state/testproj2"), testprojDir2.toPath()));
}
private String makeConfigFile(
int port,
int apiPort

4
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
View file

@ -14,6 +14,7 @@ import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import java.io.IOException;
import java.util.ArrayDeque;
@ -91,6 +92,9 @@ public class BridgeTest {
when(repoStore.getExistingRepo("asdf")).thenReturn(repo);
when(dbStore.getProjectState("asdf")).thenReturn(ProjectState.PRESENT);
when(snapshotAPI.projectExists(Optional.empty(), "asdf")).thenReturn(true);
when(
snapshotAPI.getDoc(Optional.empty(), "asdf")
).thenReturn(Optional.of(mock(GetDocResult.class)));
when(
snapshotAPI.getSnapshots(
any(),

83
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/canMigrateRepository/state/state.json Normal file
View file

@ -0,0 +1,83 @@
[
{
"project": "testproj2",
"getDoc": {
"versionID": 1,
"createdAt": "2014-11-30T18:40:58.123Z",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1",
"migratedFromId": "testproj"
},
"getSavedVers": [],
"getForVers": [
{
"versionID": 1,
"srcs": [
{
"content": "content\n",
"path": "main.tex"
},
{
"content": "This text is from another file.",
"path": "foo/bar/test.tex"
}
],
"atts": [
{
"url": "http://127.0.0.1:3857/state/testproj/min_mean_wait_evm_7_eps_150dpi.png",
"path": "min_mean_wait_evm_7_eps_150dpi.png"
}
]
}
],
"push": "success",
"postback": {
"type": "success",
"versionID": 2
}
},
{
"project": "testproj",
"getDoc": {
"versionID": 1,
"createdAt": "2014-11-30T18:40:58.123Z",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1"
},
"getSavedVers": [
{
"versionID": 1,
"comment": "added more info on doc GET and error details",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1",
"createdAt": "2014-11-30T18:47:01.333Z"
}
],
"getForVers": [
{
"versionID": 1,
"srcs": [
{
"content": "content\n",
"path": "main.tex"
},
{
"content": "This text is from another file.",
"path": "foo/bar/test.tex"
}
],
"atts": [
{
"url": "http://127.0.0.1:3857/state/testproj/min_mean_wait_evm_7_eps_150dpi.png",
"path": "min_mean_wait_evm_7_eps_150dpi.png"
}
]
}
],
"push": "success",
"postback": {
"type": "success",
"versionID": 2
}
}
]

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/canMigrateRepository/state/testproj/foo/bar/test.tex Normal file
View file

@ -0,0 +1 @@
This text is from another file.

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/canMigrateRepository/state/testproj/main.tex Normal file
View file

@ -0,0 +1 @@
content

BIN
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/canMigrateRepository/state/testproj/min_mean_wait_evm_7_eps_150dpi.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 10 KiB

39
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/skipMigrationWhenMigratedFromMissing/state/state.json Normal file
View file

@ -0,0 +1,39 @@
[
{
"project": "testproj2",
"getDoc": {
"versionID": 1,
"createdAt": "2014-11-30T18:40:58.123Z",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1",
"migratedFromId": "testprojthatdoesnotexist"
},
"getSavedVers": [],
"getForVers": [
{
"versionID": 1,
"srcs": [
{
"content": "two\n",
"path": "main.tex"
},
{
"content": "This text is from another file.",
"path": "foo/bar/test.tex"
}
],
"atts": [
{
"url": "http://127.0.0.1:3857/state/testproj/min_mean_wait_evm_7_eps_150dpi.png",
"path": "min_mean_wait_evm_7_eps_150dpi.png"
}
]
}
],
"push": "success",
"postback": {
"type": "success",
"versionID": 2
}
}
]

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/skipMigrationWhenMigratedFromMissing/state/testproj2/foo/bar/test.tex Normal file
View file

@ -0,0 +1 @@
This text is from another file.

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/skipMigrationWhenMigratedFromMissing/state/testproj2/main.tex Normal file
View file

@ -0,0 +1 @@
two

BIN
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/skipMigrationWhenMigratedFromMissing/state/testproj2/min_mean_wait_evm_7_eps_150dpi.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 10 KiB