github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #19 from overleaf/decouple-refactor

Browse source 
Decouple and Refactor

Separating a couple of commits from #17 to make it more manageable.

- Decouple jgit's Repository class from the Bridge
- Large refactor of parts into distinct components / interfaces
This commit is contained in:
Michael Mazour 2016-09-06 08:42:31 +01:00 committed by GitHub
commit 1850689a63
44 changed files with 1101 additions and 671 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
services/git-bridge/pom.xml
View file

@ -134,5 +134,11 @@
<version>3.10.4</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.mockito/mockito-core -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>1.10.19</version>
</dependency>
</dependencies>
</project>

363
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java Normal file
View file

@ -0,0 +1,363 @@
package uk.ac.ic.wlgitbridge.bridge;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobImpl;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
import uk.ac.ic.wlgitbridge.snapshot.push.PostbackManager;
import uk.ac.ic.wlgitbridge.snapshot.push.PushRequest;
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.util.*;
/**
* Created by Winston on 16/11/14.
*/
public class Bridge {
private final ProjectLock lock;
private final RepoStore repoStore;
private final DBStore dbStore;
private final SwapStore swapStore;
private final SnapshotAPI snapshotAPI;
private final ResourceCache resourceCache;
private final SwapJob swapJob;
private final PostbackManager postbackManager;
public static Bridge make(
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
ProjectLock lock = new ProjectLockImpl((int threads) ->
Log.info("Waiting for " + threads + " projects...")
);
return new Bridge(
lock,
repoStore,
dbStore,
swapStore,
new NetSnapshotAPI(),
new UrlResourceCache(dbStore),
new SwapJobImpl(
lock,
repoStore,
dbStore,
swapStore
)
);
}
Bridge(
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore,
SnapshotAPI snapshotAPI,
ResourceCache resourceCache,
SwapJob swapJob
) {
this.lock = lock;
this.repoStore = repoStore;
this.dbStore = dbStore;
this.swapStore = swapStore;
this.snapshotAPI = snapshotAPI;
this.resourceCache = resourceCache;
this.swapJob = swapJob;
postbackManager = new PostbackManager();
Runtime.getRuntime().addShutdownHook(new Thread(this::doShutdown));
repoStore.purgeNonexistentProjects(dbStore.getProjectNames());
}
void doShutdown() {
Log.info("Shutdown received.");
Log.info("Stopping SwapJob");
swapJob.stop();
Log.info("Waiting for projects");
lock.lockAll();
Log.info("Bye");
}
public void startSwapJob(int intervalMillis) {
swapJob.start(intervalMillis);
}
/* TODO: Remove these when WLBridged is moved into RepoStore */
public void lockForProject(String projectName) {
lock.lockForProject(projectName);
}
public void unlockForProject(String projectName) {
lock.unlockForProject(projectName);
}
public boolean repositoryExists(Credential oauth2, String projectName)
throws ServiceMayNotContinueException, GitUserException {
lockForProject(projectName);
GetDocRequest getDocRequest = new GetDocRequest(oauth2, projectName);
getDocRequest.request();
try {
getDocRequest.getResult().getVersionID();
} catch (InvalidProjectException e) {
return false;
} finally {
unlockForProject(projectName);
}
return true;
}
public void getWritableRepositories(
Credential oauth2,
ProjectRepo repo
) throws IOException,
GitUserException {
Log.info("[{}] Fetching", repo.getProjectName());
updateProjectWithName(oauth2, repo);
}
public void
putDirectoryContentsToProjectWithName(Credential oauth2,
String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents,
String hostname)
throws SnapshotPostException, IOException, ForbiddenException {
lock.lockForProject(projectName);
CandidateSnapshot candidate = null;
try {
Log.info("[{}] Pushing", projectName);
String postbackKey = postbackManager.makeKeyForProject(projectName);
Log.info(
"[{}] Created postback key: {}",
projectName,
postbackKey
);
candidate =
createCandidateSnapshot(
projectName,
directoryContents,
oldDirectoryContents
);
Log.info(
"[{}] Candindate snapshot created: {}",
projectName,
candidate
);
PushRequest pushRequest = new PushRequest(
oauth2,
candidate,
postbackKey
);
pushRequest.request();
PushResult result = pushRequest.getResult();
if (result.wasSuccessful()) {
Log.info(
"[{}] Push to Overleaf successful",
projectName
);
Log.info("[{}] Waiting for postback...", projectName);
int versionID =
postbackManager.waitForVersionIdOrThrow(projectName);
Log.info(
"[{}] Got version ID for push: {}",
projectName,
versionID
);
approveSnapshot(versionID, candidate);
Log.info(
"[{}] Approved version ID: {}",
projectName,
versionID
);
} else {
Log.warn(
"[{}] Went out of date while waiting for push",
projectName
);
throw new OutOfDateException();
}
} catch (SevereSnapshotPostException e) {
Log.warn("[" + projectName + "] Failed to put to Overleaf", e);
throw e;
} catch (SnapshotPostException e) {
/* Stack trace should be printed further up */
Log.warn(
"[{}] Exception when waiting for postback: {}",
projectName,
e.getClass().getSimpleName()
);
throw e;
} catch (IOException e) {
Log.warn("[{}] IOException on put", projectName);
throw e;
} finally {
if (candidate != null) {
candidate.deleteServletFiles();
} else {
Log.error(
"[{}] Candidate snapshot was null: " +
"this should never happen.",
projectName
);
}
lock.unlockForProject(projectName);
}
}
public void checkPostbackKey(String projectName, String postbackKey)
throws InvalidPostbackKeyException {
postbackManager.checkPostbackKey(projectName, postbackKey);
}
/* Called by postback thread. */
public void postbackReceivedSuccessfully(String projectName,
String postbackKey,
int versionID)
throws UnexpectedPostbackException {
Log.info(
"[{}]" +
" Postback received by postback thread, version: {}",
projectName,
versionID);
postbackManager.postVersionIDForProject(
projectName,
versionID,
postbackKey
);
}
public void postbackReceivedWithException(String projectName,
String postbackKey,
SnapshotPostException exception)
throws UnexpectedPostbackException {
Log.warn("[{}] Postback received with exception", projectName);
postbackManager.postExceptionForProject(
projectName,
exception,
postbackKey
);
}
/* PRIVATE */
private void updateProjectWithName(
Credential oauth2,
ProjectRepo repo
) throws IOException, GitUserException {
String projectName = repo.getProjectName();
Deque<Snapshot> snapshots =
snapshotAPI.getSnapshotsForProjectAfterVersion(
oauth2,
projectName,
dbStore.getLatestVersionForProject(projectName)
);
makeCommitsFromSnapshots(repo, snapshots);
if (!snapshots.isEmpty()) {
dbStore.setLatestVersionForProject(
projectName,
snapshots.getLast().getVersionID()
);
}
}
private void makeCommitsFromSnapshots(ProjectRepo repo,
Collection<Snapshot> snapshots)
throws IOException, GitUserException {
String name = repo.getProjectName();
for (Snapshot snapshot : snapshots) {
Map<String, RawFile> fileTable = repo.getFiles();
List<RawFile> files = new LinkedList<>();
files.addAll(snapshot.getSrcs());
Map<String, byte[]> fetchedUrls = new HashMap<>();
for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
files.add(
resourceCache.get(
name,
snapshotAttachment.getUrl(),
snapshotAttachment.getPath(),
fileTable,
fetchedUrls
)
);
}
Log.info(
"[{}] Committing version ID: {}",
name,
snapshot.getVersionID()
);
Collection<String> missingFiles = repo.commitAndGetMissing(
new GitDirectoryContents(
files,
repoStore.getRootDirectory(),
name,
snapshot
)
);
dbStore.deleteFilesForProject(
name,
missingFiles.toArray(new String[missingFiles.size()])
);
}
}
private CandidateSnapshot createCandidateSnapshot(
String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) throws IOException {
CandidateSnapshot candidateSnapshot = new CandidateSnapshot(
projectName,
dbStore.getLatestVersionForProject(projectName),
directoryContents,
oldDirectoryContents
);
candidateSnapshot.writeServletFiles(repoStore.getRootDirectory());
return candidateSnapshot;
}
private void approveSnapshot(
int versionID,
CandidateSnapshot candidateSnapshot
) {
List<String> deleted = candidateSnapshot.getDeleted();
dbStore.setLatestVersionForProject(
candidateSnapshot.getProjectName(),
versionID
);
dbStore.deleteFilesForProject(
candidateSnapshot.getProjectName(),
deleted.toArray(new String[deleted.size()])
);
}
}

201
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/BridgeAPI.java
View file

@ -1,201 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.data.ProjectLock;
import uk.ac.ic.wlgitbridge.data.ShutdownHook;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.data.model.DataStore;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException;
import uk.ac.ic.wlgitbridge.snapshot.push.PostbackManager;
import uk.ac.ic.wlgitbridge.snapshot.push.PushRequest;
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
/**
* Created by Winston on 16/11/14.
*/
public class BridgeAPI {
private final DataStore dataStore;
private final PostbackManager postbackManager;
private final ProjectLock mainProjectLock;
public BridgeAPI(String rootGitDirectoryPath) {
dataStore = new DataStore(rootGitDirectoryPath);
postbackManager = new PostbackManager();
mainProjectLock = new ProjectLock();
Runtime.getRuntime().addShutdownHook(new ShutdownHook(mainProjectLock));
}
public void lockForProject(String projectName) {
mainProjectLock.lockForProject(projectName);
}
public void unlockForProject(String projectName) {
mainProjectLock.unlockForProject(projectName);
}
public boolean repositoryExists(Credential oauth2, String projectName)
throws ServiceMayNotContinueException, ForbiddenException {
lockForProject(projectName);
GetDocRequest getDocRequest = new GetDocRequest(oauth2, projectName);
getDocRequest.request();
try {
getDocRequest.getResult().getVersionID();
} catch (InvalidProjectException e) {
return false;
} catch (FailedConnectionException e) {
throw e;
} catch (SnapshotPostException e) {
throw new ServiceMayNotContinueException(e.getMessage());
} finally {
unlockForProject(projectName);
}
return true;
}
public void getWritableRepositories(Credential oauth2,
String projectName,
Repository repository)
throws IOException,
SnapshotPostException,
GitAPIException,
ForbiddenException {
Log.info("[{}] Fetching", projectName);
dataStore.updateProjectWithName(oauth2, projectName, repository);
}
public void
putDirectoryContentsToProjectWithName(Credential oauth2,
String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents,
String hostname)
throws SnapshotPostException, IOException, ForbiddenException {
mainProjectLock.lockForProject(projectName);
CandidateSnapshot candidate = null;
try {
Log.info("[{}] Pushing", projectName);
String postbackKey = postbackManager.makeKeyForProject(projectName);
Log.info(
"[{}] Created postback key: {}",
projectName,
postbackKey
);
candidate =
dataStore.createCandidateSnapshot(
projectName,
directoryContents,
oldDirectoryContents
);
Log.info(
"[{}] Candindate snapshot created: {}",
projectName,
candidate
);
PushRequest pushRequest = new PushRequest(
oauth2,
candidate,
postbackKey
);
pushRequest.request();
PushResult result = pushRequest.getResult();
if (result.wasSuccessful()) {
Log.info(
"[{}] Push to Overleaf successful",
projectName
);
Log.info("[{}] Waiting for postback...", projectName);
int versionID =
postbackManager.waitForVersionIdOrThrow(projectName);
Log.info(
"[{}] Got version ID for push: {}",
projectName,
versionID
);
dataStore.approveSnapshot(versionID, candidate);
Log.info(
"[{}] Approved version ID: {}",
projectName,
versionID
);
} else {
Log.warn(
"[{}] Went out of date while waiting for push",
projectName
);
throw new OutOfDateException();
}
} catch (SevereSnapshotPostException e) {
Log.warn("[" + projectName + "] Failed to put to Overleaf", e);
throw e;
} catch (SnapshotPostException e) {
/* Stack trace should be printed further up */
Log.warn(
"[{}] Exception when waiting for postback: {}",
projectName,
e.getClass().getSimpleName()
);
throw e;
} catch (IOException e) {
Log.warn("[{}] IOException on put", projectName);
throw e;
} finally {
if (candidate != null) {
candidate.deleteServletFiles();
} else {
Log.error(
"[{}] Candidate snapshot was null: " +
"this should never happen.",
projectName
);
}
mainProjectLock.unlockForProject(projectName);
}
}
public void checkPostbackKey(String projectName, String postbackKey)
throws InvalidPostbackKeyException {
postbackManager.checkPostbackKey(projectName, postbackKey);
}
/* Called by postback thread. */
public void postbackReceivedSuccessfully(String projectName,
String postbackKey,
int versionID)
throws UnexpectedPostbackException {
Log.info(
"[{}]" +
" Postback received by postback thread, version: {}",
projectName,
versionID);
postbackManager.postVersionIDForProject(
projectName,
versionID,
postbackKey
);
}
public void postbackReceivedWithException(String projectName,
String postbackKey,
SnapshotPostException exception)
throws UnexpectedPostbackException {
Log.warn("[{}] Postback received with exception", projectName);
postbackManager.postExceptionForProject(
projectName,
exception,
postbackKey
);
}
}

93
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/GitProjectRepo.java Normal file
View file

@ -0,0 +1,93 @@
package uk.ac.ic.wlgitbridge.bridge;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
/**
* Created by winston on 20/08/2016.
*/
public class GitProjectRepo implements ProjectRepo {
private final Repository repository;
private final String projectName;
public GitProjectRepo(Repository repository, String projectName) {
this.repository = repository;
this.projectName = projectName;
}
@Override
public String getProjectName() {
return projectName;
}
@Override
public Map<String, RawFile> getFiles()
throws IOException, GitUserException {
return new RepositoryObjectTreeWalker(
repository
).getDirectoryContents().getFileTable();
}
@Override
public Collection<String> commitAndGetMissing(
GitDirectoryContents contents
) throws IOException {
try {
return doCommitAndGetMissing(contents);
} catch (GitAPIException e) {
throw new IOException(e);
}
}
private Collection<String> doCommitAndGetMissing(
GitDirectoryContents contents
) throws IOException, GitAPIException {
String name = getProjectName();
Log.info("[{}] Writing commit", name);
contents.write();
Git git = new Git(repository);
Log.info("[{}] Getting missing files", name);
Set<String> missingFiles = git.status().call().getMissing();
for (String missing : missingFiles) {
Log.info("[{}] Git rm {}", name, missing);
git.rm().setCached(true).addFilepattern(missing).call();
}
Log.info("[{}] Calling Git add", name);
git.add().addFilepattern(".").call();
Log.info("[{}] Calling Git commit", name);
git.commit(
).setAuthor(
new PersonIdent(
contents.getUserName(),
contents.getUserEmail(),
contents.getWhen(),
TimeZone.getDefault()
)
).setMessage(
contents.getCommitMessage()
).call();
Log.info(
"[{}] Deleting files in directory: {}",
name,
contents.getDirectory().getAbsolutePath()
);
Util.deleteInDirectoryApartFrom(contents.getDirectory(), ".git");
return missingFiles;
}
}

24
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/ProjectRepo.java Normal file
View file

@ -0,0 +1,24 @@
package uk.ac.ic.wlgitbridge.bridge;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
/**
* Created by winston on 20/08/2016.
*/
public interface ProjectRepo {
String getProjectName();
Map<String, RawFile> getFiles() throws IOException, GitUserException;
Collection<String> commitAndGetMissing(
GitDirectoryContents gitDirectoryContents
) throws IOException;
}

28
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/WLBridgedProject.java
View file

@ -1,13 +1,11 @@
package uk.ac.ic.wlgitbridge.bridge;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import java.io.IOException;
@ -18,15 +16,15 @@ public class WLBridgedProject {
private final Repository repository;
private final String name;
private final BridgeAPI bridgeAPI;
private final Bridge bridgeAPI;
public WLBridgedProject(Repository repository, String name, BridgeAPI bridgeAPI) {
public WLBridgedProject(Repository repository, String name, Bridge bridgeAPI) {
this.repository = repository;
this.name = name;
this.bridgeAPI = bridgeAPI;
}
public void buildRepository(Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, ForbiddenException {
public void buildRepository(Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException {
bridgeAPI.lockForProject(name);
try {
if (repository.getObjectDatabase().exists()) {
@ -42,21 +40,25 @@ public class WLBridgedProject {
}
}
private void updateRepositoryFromSnapshots(Credential oauth2, Repository repository) throws RepositoryNotFoundException, ServiceMayNotContinueException, ForbiddenException {
private void updateRepositoryFromSnapshots(
Credential oauth2,
Repository repository
) throws RepositoryNotFoundException,
ServiceMayNotContinueException,
GitUserException {
try {
bridgeAPI.getWritableRepositories(oauth2, name, repository);
bridgeAPI.getWritableRepositories(
oauth2,
new GitProjectRepo(repository, name)
);
} catch (InvalidProjectException e) {
throw new RepositoryNotFoundException(name);
} catch (SnapshotPostException e) {
throw new ServiceMayNotContinueException(e.getDescriptionLines().get(0), e);
} catch (GitAPIException e) {
throw new ServiceMayNotContinueException(e);
} catch (IOException e) {
throw new ServiceMayNotContinueException(e);
}
}
private void buildRepositoryFromScratch(Credential oauth2, Repository repository) throws RepositoryNotFoundException, ServiceMayNotContinueException, ForbiddenException {
private void buildRepositoryFromScratch(Credential oauth2, Repository repository) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException {
if (!bridgeAPI.repositoryExists(oauth2, name)) {
throw new RepositoryNotFoundException(name);
}

22
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java Normal file
View file

@ -0,0 +1,22 @@
package uk.ac.ic.wlgitbridge.bridge.db;
import java.util.List;
/**
* Created by winston on 20/08/2016.
*/
public interface DBStore {
List<String> getProjectNames();
void setLatestVersionForProject(String project, int versionID);
int getLatestVersionForProject(String project);
void addURLIndexForProject(String projectName, String url, String path);
void deleteFilesForProject(String project, String... files);
String getPathForURLInProject(String projectName, String url);
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/SqlitePersistentStore.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java
View file

@ -1,4 +1,4 @@
package uk.ac.ic.wlgitbridge.data.model.db;
package uk.ac.ic.wlgitbridge.bridge.db;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLiteWLDatabase;
import uk.ac.ic.wlgitbridge.util.Log;
@ -9,15 +9,15 @@ import java.util.Arrays;
import java.util.List;
/**
* Created by Winston on 19/11/14.
* Created by winston on 20/08/2016.
*/
public class SqlitePersistentStore implements PersistentStore {
public class SqliteDBStore implements DBStore {
private final SQLiteWLDatabase database;
public SqlitePersistentStore(File rootGitDirectory) {
public SqliteDBStore(File rootDirectory) {
try {
database = new SQLiteWLDatabase(rootGitDirectory);
database = new SQLiteWLDatabase(rootDirectory);
} catch (SQLException e) {
throw new RuntimeException(e);
} catch (ClassNotFoundException e) {

12
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java Normal file
View file

@ -0,0 +1,12 @@
package uk.ac.ic.wlgitbridge.bridge.lock;
/**
* Created by winston on 20/08/2016.
*/
public interface ProjectLock {
void lockAll();
void lockForProject(String projectName);
void unlockForProject(String projectName);
}

52
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java Normal file
View file

@ -0,0 +1,52 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/**
* Created by winston on 20/08/2016.
*/
public class FSRepoStore implements RepoStore {
private final String repoStorePath;
private final File rootDirectory;
public FSRepoStore(String repoStorePath) {
this.repoStorePath = repoStorePath;
rootDirectory = initRootGitDirectory(repoStorePath);
}
@Override
public String getRepoStorePath() {
return repoStorePath;
}
@Override
public File getRootDirectory() {
return rootDirectory;
}
@Override
public void purgeNonexistentProjects(
Collection<String> existingProjectNames
) {
List<String> excludedFromDeletion =
new ArrayList<>(existingProjectNames);
excludedFromDeletion.add(".wlgb");
Util.deleteInDirectoryApartFrom(
rootDirectory,
excludedFromDeletion.toArray(new String[] {})
);
}
private File initRootGitDirectory(String rootGitDirectoryPath) {
File rootGitDirectory = new File(rootGitDirectoryPath);
rootGitDirectory.mkdirs();
return rootGitDirectory;
}
}

22
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java Normal file
View file

@ -0,0 +1,22 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import java.io.File;
import java.util.Collection;
/**
* Created by winston on 20/08/2016.
*/
public interface RepoStore {
String getRepoStorePath();
File getRootDirectory();
void purgeNonexistentProjects(
Collection<String> existingProjectNames
);
}

13
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java Normal file
View file

@ -0,0 +1,13 @@
package uk.ac.ic.wlgitbridge.bridge.resource;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import java.io.IOException;
import java.util.Map;
/**
* Created by winston on 20/08/2016.
*/
public interface ResourceCache {
RawFile get(String projectName, String url, String newPath, Map<String, RawFile> fileTable, Map<String, byte[]> fetchedUrls) throws IOException;
}

107
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java Normal file
View file

@ -0,0 +1,107 @@
package uk.ac.ic.wlgitbridge.bridge.resource;
import com.ning.http.client.AsyncCompletionHandler;
import com.ning.http.client.HttpResponseBodyPart;
import com.ning.http.client.Response;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
import uk.ac.ic.wlgitbridge.snapshot.base.Request;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ExecutionException;
/**
* Created by winston on 20/08/2016.
*/
public class UrlResourceCache implements ResourceCache {
private final DBStore dbStore;
public UrlResourceCache(DBStore dbStore) {
this.dbStore = dbStore;
}
@Override
public RawFile get(String projectName, String url, String newPath, Map<String, RawFile> fileTable, Map<String, byte[]> fetchedUrls) throws IOException {
String path = dbStore.getPathForURLInProject(projectName, url);
byte[] contents;
if (path == null) {
path = newPath;
contents = fetch(projectName, url, path);
fetchedUrls.put(url, contents);
} else {
Log.info("Found (" + projectName + "): " + url);
Log.info("At (" + projectName + "): " + path);
contents = fetchedUrls.get(url);
if (contents == null) {
RawFile rawFile = fileTable.get(path);
if (rawFile == null) {
Log.warn(
"File " + path + " was not in the current commit, or the git tree, yet path was not null. " +
"File url is: " + url
);
contents = fetch(projectName, url, path);
} else {
contents = rawFile.getContents();
}
}
}
return new RepositoryFile(newPath, contents);
}
private byte[] fetch(String projectName, final String url, String path) throws FailedConnectionException {
byte[] contents;
Log.info("GET -> " + url);
try {
contents = Request.httpClient.prepareGet(url).execute(new AsyncCompletionHandler<byte[]>() {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@Override
public STATE onBodyPartReceived(HttpResponseBodyPart bodyPart) throws Exception {
bytes.write(bodyPart.getBodyPartBytes());
return STATE.CONTINUE;
}
@Override
public byte[] onCompleted(Response response) throws Exception {
byte[] data = bytes.toByteArray();
bytes.close();
Log.info(response.getStatusCode() + " " + response.getStatusText() + " (" + data.length + "B) -> " + url);
return data;
}
}).get();
} catch (InterruptedException e) {
Log.warn(
"Interrupted when fetching project: " +
projectName +
", url: " +
url +
", path: " +
path,
e
);
throw new FailedConnectionException();
} catch (ExecutionException e) {
Log.warn(
"ExecutionException when fetching project: " +
projectName +
", url: " +
url +
", path: " +
path,
e
);
throw new FailedConnectionException();
}
dbStore.addURLIndexForProject(projectName, url, path);
return contents;
}
}

79
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotAPI.java Normal file
View file

@ -0,0 +1,79 @@
package uk.ac.ic.wlgitbridge.bridge.snapshot;
import com.google.api.client.auth.oauth2.Credential;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionRequest;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersRequest;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
import java.util.*;
/**
* Created by winston on 20/08/2016.
*/
public class NetSnapshotAPI implements SnapshotAPI {
@Override
public Deque<Snapshot> getSnapshotsForProjectAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, GitUserException {
List<SnapshotInfo> snapshotInfos = getSnapshotInfosAfterVersion(oauth2, projectName, version);
List<SnapshotData> snapshotDatas = getMatchingSnapshotData(oauth2, projectName, snapshotInfos);
LinkedList<Snapshot> snapshots = combine(snapshotInfos, snapshotDatas);
return snapshots;
}
private List<SnapshotInfo> getSnapshotInfosAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, GitUserException {
SortedSet<SnapshotInfo> versions = new TreeSet<SnapshotInfo>();
GetDocRequest getDoc = new GetDocRequest(oauth2, projectName);
GetSavedVersRequest getSavedVers = new GetSavedVersRequest(oauth2, projectName);
getDoc.request();
getSavedVers.request();
GetDocResult latestDoc = getDoc.getResult();
int latest = latestDoc.getVersionID();
if (latest > version) {
for (SnapshotInfo snapshotInfo : getSavedVers.getResult().getSavedVers()) {
if (snapshotInfo.getVersionId() > version) {
versions.add(snapshotInfo);
}
}
versions.add(new SnapshotInfo(latest, latestDoc.getCreatedAt(), latestDoc.getName(), latestDoc.getEmail()));
}
return new LinkedList<SnapshotInfo>(versions);
}
private List<SnapshotData> getMatchingSnapshotData(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) throws FailedConnectionException, ForbiddenException {
List<GetForVersionRequest> firedRequests = fireDataRequests(oauth2, projectName, snapshotInfos);
List<SnapshotData> snapshotDataList = new LinkedList<SnapshotData>();
for (GetForVersionRequest fired : firedRequests) {
snapshotDataList.add(fired.getResult().getSnapshotData());
}
return snapshotDataList;
}
private List<GetForVersionRequest> fireDataRequests(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) {
List<GetForVersionRequest> requests = new LinkedList<GetForVersionRequest>();
for (SnapshotInfo snapshotInfo : snapshotInfos) {
GetForVersionRequest request = new GetForVersionRequest(oauth2, projectName, snapshotInfo.getVersionId());
requests.add(request);
request.request();
}
return requests;
}
private LinkedList<Snapshot> combine(List<SnapshotInfo> snapshotInfos, List<SnapshotData> snapshotDatas) {
LinkedList<Snapshot> snapshots = new LinkedList<Snapshot>();
Iterator<SnapshotInfo> infos = snapshotInfos.iterator();
Iterator<SnapshotData> datas = snapshotDatas.iterator();
while (infos.hasNext()) {
snapshots.add(new Snapshot(infos.next(), datas.next()));
}
return snapshots;
}
}

21
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotAPI.java Normal file
View file

@ -0,0 +1,21 @@
package uk.ac.ic.wlgitbridge.bridge.snapshot;
import com.google.api.client.auth.oauth2.Credential;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import java.util.Deque;
/**
* Created by winston on 20/08/2016.
*/
public interface SnapshotAPI {
Deque<Snapshot> getSnapshotsForProjectAfterVersion(
Credential oauth2,
String projectName,
int latestVersion
) throws FailedConnectionException, GitUserException;
}

12
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java Normal file
View file

@ -0,0 +1,12 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapJob {
void start(int intervalMillis);
void stop();
}

53
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java Normal file
View file

@ -0,0 +1,53 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.util.Util;
import java.util.Timer;
/**
* Created by winston on 20/08/2016.
*/
public class SwapJobImpl implements SwapJob {
private final ProjectLock lock;
private final RepoStore repoStore;
private final SwapStore swapStore;
private final DBStore dbStore;
private final Timer timer;
public SwapJobImpl(
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore, SwapStore swapStore
) {
this.lock = lock;
this.repoStore = repoStore;
this.swapStore = swapStore;
this.dbStore = dbStore;
timer = new Timer();
}
@Override
public void start(int intervalMillis) {
timer.scheduleAtFixedRate(
Util.makeTimerTask(this::doSwap),
0,
intervalMillis
);
}
@Override
public void stop() {
timer.cancel();
}
private void doSwap() {
throw new UnsupportedOperationException();
}
}

7
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java Normal file
View file

@ -0,0 +1,7 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapStore {
}

11
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLock.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLockImpl.java
View file

@ -1,5 +1,7 @@
package uk.ac.ic.wlgitbridge.data;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.locks.Lock;
@ -9,7 +11,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* Created by Winston on 20/11/14.
*/
public class ProjectLock {
public class ProjectLockImpl implements ProjectLock {
private final Map<String, Lock> projectLocks;
private final ReentrantReadWriteLock rwlock;
@ -18,7 +20,7 @@ public class ProjectLock {
private LockAllWaiter waiter;
private boolean waiting;
public ProjectLock() {
public ProjectLockImpl() {
projectLocks = new HashMap<String, Lock>();
rwlock = new ReentrantReadWriteLock();
rlock = rwlock.readLock();
@ -26,6 +28,11 @@ public class ProjectLock {
waiting = false;
}
public ProjectLockImpl(LockAllWaiter waiter) {
this();
setWaiter(waiter);
}
public void lockForProject(String projectName) {
getLockForProjectName(projectName).lock();
rlock.lock();

30
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ShutdownHook.java
View file

@ -1,30 +0,0 @@
package uk.ac.ic.wlgitbridge.data;
import uk.ac.ic.wlgitbridge.util.Log;
/**
* Created by Winston on 21/02/15.
*/
public class ShutdownHook extends Thread implements LockAllWaiter {
private final ProjectLock projectLock;
public ShutdownHook(ProjectLock projectLock) {
this.projectLock = projectLock;
projectLock.setWaiter(this);
}
@Override
public void run() {
Log.info("Shutdown received.");
projectLock.lockAll();
Log.info("No projects to wait for.");
Log.info("Bye");
}
@Override
public void threadsRemaining(int threads) {
Log.info("Waiting for " + threads + " projects...");
}
}

68
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/SnapshotFetcher.java
View file

@ -1,78 +1,10 @@
package uk.ac.ic.wlgitbridge.data;
import com.google.api.client.auth.oauth2.Credential;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionRequest;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersRequest;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import java.util.*;
/**
* Created by Winston on 07/11/14.
*/
public class SnapshotFetcher {
public LinkedList<Snapshot> getSnapshotsForProjectAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, SnapshotPostException, ForbiddenException {
List<SnapshotInfo> snapshotInfos = getSnapshotInfosAfterVersion(oauth2, projectName, version);
List<SnapshotData> snapshotDatas = getMatchingSnapshotData(oauth2, projectName, snapshotInfos);
LinkedList<Snapshot> snapshots = combine(snapshotInfos, snapshotDatas);
return snapshots;
}
private List<SnapshotInfo> getSnapshotInfosAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, SnapshotPostException, ForbiddenException {
SortedSet<SnapshotInfo> versions = new TreeSet<SnapshotInfo>();
GetDocRequest getDoc = new GetDocRequest(oauth2, projectName);
GetSavedVersRequest getSavedVers = new GetSavedVersRequest(oauth2, projectName);
getDoc.request();
getSavedVers.request();
GetDocResult latestDoc = getDoc.getResult();
int latest = latestDoc.getVersionID();
if (latest > version) {
for (SnapshotInfo snapshotInfo : getSavedVers.getResult().getSavedVers()) {
if (snapshotInfo.getVersionId() > version) {
versions.add(snapshotInfo);
}
}
versions.add(new SnapshotInfo(latest, latestDoc.getCreatedAt(), latestDoc.getName(), latestDoc.getEmail()));
}
return new LinkedList<SnapshotInfo>(versions);
}
private List<SnapshotData> getMatchingSnapshotData(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) throws FailedConnectionException, ForbiddenException {
List<GetForVersionRequest> firedRequests = fireDataRequests(oauth2, projectName, snapshotInfos);
List<SnapshotData> snapshotDataList = new LinkedList<SnapshotData>();
for (GetForVersionRequest fired : firedRequests) {
snapshotDataList.add(fired.getResult().getSnapshotData());
}
return snapshotDataList;
}
private List<GetForVersionRequest> fireDataRequests(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) {
List<GetForVersionRequest> requests = new LinkedList<GetForVersionRequest>();
for (SnapshotInfo snapshotInfo : snapshotInfos) {
GetForVersionRequest request = new GetForVersionRequest(oauth2, projectName, snapshotInfo.getVersionId());
requests.add(request);
request.request();
}
return requests;
}
private LinkedList<Snapshot> combine(List<SnapshotInfo> snapshotInfos, List<SnapshotData> snapshotDatas) {
LinkedList<Snapshot> snapshots = new LinkedList<Snapshot>();
Iterator<SnapshotInfo> infos = snapshotInfos.iterator();
Iterator<SnapshotData> datas = snapshotDatas.iterator();
while (infos.hasNext()) {
snapshots.add(new Snapshot(infos.next(), datas.next()));
}
return snapshots;
}
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/SnapshotRepositoryBuilder.java
View file

@ -5,9 +5,9 @@ import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.WLBridgedProject;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException;
import uk.ac.ic.wlgitbridge.util.Log;
@ -19,13 +19,13 @@ import java.io.IOException;
*/
public class SnapshotRepositoryBuilder {
private final BridgeAPI bridgeAPI;
private final Bridge bridgeAPI;
public SnapshotRepositoryBuilder(BridgeAPI bridgeAPI) {
public SnapshotRepositoryBuilder(Bridge bridgeAPI) {
this.bridgeAPI = bridgeAPI;
}
public Repository getRepositoryWithNameAtRootDirectory(String name, File rootDirectory, Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, ForbiddenException {
public Repository getRepositoryWithNameAtRootDirectory(String name, File rootDirectory, Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException {
if (!bridgeAPI.repositoryExists(oauth2, name)) {
throw new RepositoryNotFoundException(name);
}

183
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/DataStore.java
View file

@ -1,183 +0,0 @@
package uk.ac.ic.wlgitbridge.data.model;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.data.SnapshotFetcher;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
import uk.ac.ic.wlgitbridge.data.model.db.SqlitePersistentStore;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.io.IOException;
import java.util.*;
/**
* Created by Winston on 06/11/14.
*/
public class DataStore {
private final File rootGitDirectory;
private final PersistentStore persistentStore;
private final SnapshotFetcher snapshotFetcher;
private final ResourceFetcher resourceFetcher;
public DataStore(String rootGitDirectoryPath) {
rootGitDirectory = initRootGitDirectory(rootGitDirectoryPath);
persistentStore = new SqlitePersistentStore(rootGitDirectory);
List<String> excludedFromDeletion = persistentStore.getProjectNames();
excludedFromDeletion.add(".wlgb");
Util.deleteInDirectoryApartFrom(
rootGitDirectory,
excludedFromDeletion.toArray(new String[] {})
);
snapshotFetcher = new SnapshotFetcher();
resourceFetcher = new ResourceFetcher(persistentStore);
}
public void updateProjectWithName(Credential oauth2,
String name,
Repository repository)
throws IOException,
SnapshotPostException,
GitAPIException,
ForbiddenException {
LinkedList<Snapshot> snapshots =
snapshotFetcher.getSnapshotsForProjectAfterVersion(
oauth2,
name,
persistentStore.getLatestVersionForProject(name)
);
makeCommitsFromSnapshots(name, repository, snapshots);
if (!snapshots.isEmpty()) {
persistentStore.setLatestVersionForProject(
name,
snapshots.getLast().getVersionID()
);
}
}
private void makeCommitsFromSnapshots(String name,
Repository repository,
List<Snapshot> snapshots)
throws IOException, GitAPIException, SnapshotPostException {
for (Snapshot snapshot : snapshots) {
Map<String, RawFile> fileTable = new RepositoryObjectTreeWalker(repository).getDirectoryContents().getFileTable();
List<RawFile> files = new LinkedList<RawFile>();
files.addAll(snapshot.getSrcs());
Map<String, byte[]> fetchedUrls = new HashMap<String, byte[]>();
for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
files.add(
resourceFetcher.get(
name,
snapshotAttachment.getUrl(),
snapshotAttachment.getPath(),
fileTable,
fetchedUrls
)
);
}
Log.info(
"[{}] Committing version ID: {}",
name,
snapshot.getVersionID()
);
commit(name,
new GitDirectoryContents(
files,
rootGitDirectory,
name,
snapshot),
repository);
}
}
private void commit(String name,
GitDirectoryContents contents,
Repository repository) throws IOException,
GitAPIException {
Log.info("[{}] Writing commit", name);
contents.write();
Git git = new Git(repository);
Log.info("[{}] Getting missing files", name);
Set<String> missingFiles = git.status().call().getMissing();
for (String missing : missingFiles) {
Log.info("[{}] Git rm {}", name, missing);
git.rm().setCached(true).addFilepattern(missing).call();
}
Log.info("[{}] Calling Git add", name);
git.add().addFilepattern(".").call();
Log.info("[{}] Calling Git commit", name);
git.commit(
).setAuthor(
new PersonIdent(
contents.getUserName(),
contents.getUserEmail(),
contents.getWhen(),
TimeZone.getDefault()
)
).setMessage(
contents.getCommitMessage()
).call();
persistentStore.deleteFilesForProject(
name,
missingFiles.toArray(new String[missingFiles.size()])
);
Log.info(
"[{}] Deleting files in directory: {}",
name,
contents.getDirectory().getAbsolutePath()
);
Util.deleteInDirectoryApartFrom(contents.getDirectory(), ".git");
}
public CandidateSnapshot
createCandidateSnapshot(String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents)
throws SnapshotPostException,
IOException {
CandidateSnapshot candidateSnapshot = new CandidateSnapshot(
projectName,
persistentStore.getLatestVersionForProject(projectName),
directoryContents,
oldDirectoryContents
);
candidateSnapshot.writeServletFiles(rootGitDirectory);
return candidateSnapshot;
}
public void approveSnapshot(int versionID,
CandidateSnapshot candidateSnapshot) {
List<String> deleted = candidateSnapshot.getDeleted();
persistentStore.setLatestVersionForProject(
candidateSnapshot.getProjectName(),
versionID
);
persistentStore.deleteFilesForProject(
candidateSnapshot.getProjectName(),
deleted.toArray(new String[deleted.size()])
);
}
private File initRootGitDirectory(String rootGitDirectoryPath) {
File rootGitDirectory = new File(rootGitDirectoryPath);
rootGitDirectory.mkdirs();
return rootGitDirectory;
}
}

99
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcher.java
View file

@ -1,108 +1,9 @@
package uk.ac.ic.wlgitbridge.data.model;
import com.ning.http.client.AsyncCompletionHandler;
import com.ning.http.client.HttpResponseBodyPart;
import com.ning.http.client.Response;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import uk.ac.ic.wlgitbridge.snapshot.base.Request;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ExecutionException;
/**
* Created by Winston on 21/02/15.
*/
public class ResourceFetcher {
private final PersistentStore persistentStore;
public ResourceFetcher(PersistentStore persistentStore) {
this.persistentStore = persistentStore;
}
public RawFile get(String projectName, String url, String newPath, Map<String, RawFile> fileTable, Map<String, byte[]> fetchedUrls) throws IOException, SnapshotPostException {
String path = persistentStore.getPathForURLInProject(projectName, url);
byte[] contents;
if (path == null) {
path = newPath;
contents = fetch(projectName, url, path);
fetchedUrls.put(url, contents);
} else {
Log.info("Found (" + projectName + "): " + url);
Log.info("At (" + projectName + "): " + path);
contents = fetchedUrls.get(url);
if (contents == null) {
RawFile rawFile = fileTable.get(path);
if (rawFile == null) {
Log.warn(
"File " + path + " was not in the current commit, or the git tree, yet path was not null. " +
"File url is: " + url
);
contents = fetch(projectName, url, path);
} else {
contents = rawFile.getContents();
}
}
}
return new RepositoryFile(newPath, contents);
}
private byte[] fetch(String projectName, final String url, String path) throws FailedConnectionException {
byte[] contents;
Log.info("GET -> " + url);
try {
contents = Request.httpClient.prepareGet(url).execute(new AsyncCompletionHandler<byte[]>() {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@Override
public STATE onBodyPartReceived(HttpResponseBodyPart bodyPart) throws Exception {
bytes.write(bodyPart.getBodyPartBytes());
return STATE.CONTINUE;
}
@Override
public byte[] onCompleted(Response response) throws Exception {
byte[] data = bytes.toByteArray();
bytes.close();
Log.info(response.getStatusCode() + " " + response.getStatusText() + " (" + data.length + "B) -> " + url);
return data;
}
}).get();
} catch (InterruptedException e) {
Log.warn(
"Interrupted when fetching project: " +
projectName +
", url: " +
url +
", path: " +
path,
e
);
throw new FailedConnectionException();
} catch (ExecutionException e) {
Log.warn(
"ExecutionException when fetching project: " +
projectName +
", url: " +
url +
", path: " +
path,
e
);
throw new FailedConnectionException();
}
persistentStore.addURLIndexForProject(projectName, url, path);
return contents;
}
}

13
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/GitUserException.java Normal file
View file

@ -0,0 +1,13 @@
package uk.ac.ic.wlgitbridge.git.exception;
import java.util.List;
/**
* Created by winston on 20/08/2016.
*/
public abstract class GitUserException extends Exception {
public abstract String getMessage();
public abstract List<String> getDescriptionLines();
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/SizeLimitExceededException.java
View file

@ -1,13 +1,11 @@
package uk.ac.ic.wlgitbridge.git.exception;
import com.google.gson.JsonElement;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import uk.ac.ic.wlgitbridge.util.Util;
import java.util.Arrays;
import java.util.List;
public class SizeLimitExceededException extends SnapshotPostException {
public class SizeLimitExceededException extends GitUserException {
private static String path = null;
@ -26,12 +24,8 @@ public class SizeLimitExceededException extends SnapshotPostException {
String filename = path != null ? "File '" + path + "' is" : "There's a file";
return Arrays.asList(
filename + " too large to push to " + Util.getServiceName() + " via git",
"the recommended maximum file size is 15MiB"
"the recommended maximum file size is 50 MiB"
);
}
@Override
public void fromJSON(JsonElement json) {
}
}

11
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/SnapshotAPIException.java Normal file
View file

@ -0,0 +1,11 @@
package uk.ac.ic.wlgitbridge.git.exception;
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
/**
* Created by winston on 20/08/2016.
*/
public abstract class SnapshotAPIException
extends GitUserException
implements JSONSource {
}

6
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java
View file

@ -6,7 +6,7 @@ import org.eclipse.jgit.transport.ReceivePack;
import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
import uk.ac.ic.wlgitbridge.util.Util;
@ -19,9 +19,9 @@ import javax.servlet.http.HttpServletRequest;
/* */
public class WLReceivePackFactory implements ReceivePackFactory<HttpServletRequest> {
private final BridgeAPI bridgeAPI;
private final Bridge bridgeAPI;
public WLReceivePackFactory(BridgeAPI bridgeAPI) {
public WLReceivePackFactory(Bridge bridgeAPI) {
this.bridgeAPI = bridgeAPI;
}

3
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLRepositoryResolver.java
View file

@ -8,6 +8,7 @@ import org.eclipse.jgit.transport.resolver.RepositoryResolver;
import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
import uk.ac.ic.wlgitbridge.data.SnapshotRepositoryBuilder;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.exception.InvalidRootDirectoryPathException;
import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
@ -51,6 +52,8 @@ public class WLRepositoryResolver implements RepositoryResolver<HttpServletReque
throw new ServiceMayNotContinueException(e);
} catch (ForbiddenException e) {
throw new ServiceNotAuthorizedException();
} catch (GitUserException e) {
throw new ServiceMayNotContinueException(e.getMessage(), e);
}
}

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java
View file

@ -6,12 +6,12 @@ import org.eclipse.jgit.transport.PreReceiveHook;
import org.eclipse.jgit.transport.ReceiveCommand;
import org.eclipse.jgit.transport.ReceiveCommand.Result;
import org.eclipse.jgit.transport.ReceivePack;
import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.handler.hook.exception.ForcedPushException;
import uk.ac.ic.wlgitbridge.git.handler.hook.exception.WrongBranchException;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.OutOfDateException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
@ -26,11 +26,11 @@ import java.util.Iterator;
*/
public class WriteLatexPutHook implements PreReceiveHook {
private final BridgeAPI bridgeAPI;
private final Bridge bridgeAPI;
private final String hostname;
private final Credential oauth2;
public WriteLatexPutHook(BridgeAPI bridgeAPI, String hostname, Credential oauth2) {
public WriteLatexPutHook(Bridge bridgeAPI, String hostname, Credential oauth2) {
this.bridgeAPI = bridgeAPI;
this.hostname = hostname;
this.oauth2 = oauth2;
@ -72,7 +72,7 @@ public class WriteLatexPutHook implements PreReceiveHook {
receiveCommand.setResult(Result.REJECTED_OTHER_REASON, message);
}
private void handleReceiveCommand(Credential oauth2, Repository repository, ReceiveCommand receiveCommand) throws IOException, SnapshotPostException, ForbiddenException {
private void handleReceiveCommand(Credential oauth2, Repository repository, ReceiveCommand receiveCommand) throws IOException, GitUserException {
checkBranch(receiveCommand);
checkForcedPush(receiveCommand);
bridgeAPI.putDirectoryContentsToProjectWithName(
@ -97,13 +97,13 @@ public class WriteLatexPutHook implements PreReceiveHook {
}
}
private RawDirectory getPushedDirectoryContents(Repository repository, ReceiveCommand receiveCommand) throws IOException, SnapshotPostException {
private RawDirectory getPushedDirectoryContents(Repository repository, ReceiveCommand receiveCommand) throws IOException, GitUserException {
return new RepositoryObjectTreeWalker(repository,
receiveCommand.getNewId())
.getDirectoryContents();
}
private RawDirectory getOldDirectoryContents(Repository repository) throws IOException, SnapshotPostException {
private RawDirectory getOldDirectoryContents(Repository repository) throws IOException, GitUserException {
return new RepositoryObjectTreeWalker(repository).getDirectoryContents();
}

4
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServlet.java
View file

@ -7,7 +7,7 @@ import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
import uk.ac.ic.wlgitbridge.git.handler.WLRepositoryResolver;
import uk.ac.ic.wlgitbridge.git.handler.WLUploadPackFactory;
import uk.ac.ic.wlgitbridge.data.SnapshotRepositoryBuilder;
import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import javax.servlet.ServletException;
@ -16,7 +16,7 @@ import javax.servlet.ServletException;
*/
public class WLGitServlet extends GitServlet {
public WLGitServlet(ServletContextHandler servletContextHandler, BridgeAPI bridgeAPI, String rootGitDirectoryPath) throws ServletException, InvalidRootDirectoryPathException {
public WLGitServlet(ServletContextHandler servletContextHandler, Bridge bridgeAPI, String rootGitDirectoryPath) throws ServletException, InvalidRootDirectoryPathException {
setRepositoryResolver(new WLRepositoryResolver(rootGitDirectoryPath, new SnapshotRepositoryBuilder(bridgeAPI)));
setReceivePackFactory(new WLReceivePackFactory(bridgeAPI));
setUploadPackFactory(new WLUploadPackFactory());

6
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java
View file

@ -8,8 +8,8 @@ import org.eclipse.jgit.treewalk.TreeWalk;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import java.io.IOException;
import java.util.HashMap;
@ -36,7 +36,7 @@ public class RepositoryObjectTreeWalker {
this(repository, repository.resolve("HEAD~" + fromHead));
}
public RawDirectory getDirectoryContents() throws IOException, SnapshotPostException {
public RawDirectory getDirectoryContents() throws IOException, GitUserException {
return new RawDirectory(walkGitObjectTree());
}
@ -51,7 +51,7 @@ public class RepositoryObjectTreeWalker {
return treeWalk;
}
private Map<String, RawFile> walkGitObjectTree() throws IOException, SnapshotPostException {
private Map<String, RawFile> walkGitObjectTree() throws IOException, GitUserException {
Map<String, RawFile> fileContentsTable = new HashMap<String, RawFile>();
if (treeWalk == null) {
return fileContentsTable;

6
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/FileHandler.java
View file

@ -4,7 +4,7 @@ import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.ResourceHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.InvalidPostbackKeyException;
import javax.servlet.ServletException;
@ -22,10 +22,10 @@ import java.util.regex.Pattern;
public class FileHandler extends ResourceHandler {
private static final Logger LOG = LoggerFactory.getLogger(FileHandler.class);
private final BridgeAPI writeLatexDataSource;
private final Bridge writeLatexDataSource;
private final Pattern DOC_KEY_PATTERN = Pattern.compile("^/(\\w+)/.+$");
public FileHandler(BridgeAPI writeLatexDataSource) {
public FileHandler(Bridge writeLatexDataSource) {
this.writeLatexDataSource = writeLatexDataSource;
}

18
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
View file

@ -8,7 +8,12 @@ import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import uk.ac.ic.wlgitbridge.application.config.Config;
import uk.ac.ic.wlgitbridge.application.jetty.NullLogger;
import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.SqliteDBStore;
import uk.ac.ic.wlgitbridge.bridge.repo.FSRepoStore;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import uk.ac.ic.wlgitbridge.git.exception.InvalidRootDirectoryPathException;
import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
import uk.ac.ic.wlgitbridge.snapshot.base.SnapshotAPIRequest;
@ -31,7 +36,7 @@ import java.util.EnumSet;
*/
public class GitBridgeServer {
private final BridgeAPI bridgeAPI;
private final Bridge bridgeAPI;
private final Server jettyServer;
@ -43,7 +48,14 @@ public class GitBridgeServer {
org.eclipse.jetty.util.log.Log.setLog(new NullLogger());
this.port = config.getPort();
this.rootGitDirectoryPath = config.getRootGitDirectory();
bridgeAPI = new BridgeAPI(rootGitDirectoryPath);
RepoStore repoStore = new FSRepoStore(rootGitDirectoryPath);
DBStore dbStore = new SqliteDBStore(repoStore.getRootDirectory());
SwapStore swapStore = new SwapStore() {};
bridgeAPI = Bridge.make(
repoStore,
dbStore,
swapStore
);
jettyServer = new Server(port);
configureJettyServer(config);
SnapshotAPIRequest.setBasicAuth(config.getUsername(), config.getPassword());

6
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/PostbackContents.java
View file

@ -3,7 +3,7 @@ package uk.ac.ic.wlgitbridge.server;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.UnexpectedPostbackException;
@ -17,7 +17,7 @@ public class PostbackContents implements JSONSource {
private static final String CODE_SUCCESS = "upToDate";
private final BridgeAPI bridgeAPI;
private final Bridge bridgeAPI;
private final String projectName;
private final String postbackKey;
@ -26,7 +26,7 @@ public class PostbackContents implements JSONSource {
private int versionID;
private SnapshotPostException exception;
public PostbackContents(BridgeAPI bridgeAPI, String projectName, String postbackKey, String contents) {
public PostbackContents(Bridge bridgeAPI, String projectName, String postbackKey, String contents) {
this.bridgeAPI = bridgeAPI;
this.projectName = projectName;
this.postbackKey = postbackKey;

6
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/PostbackHandler.java
View file

@ -4,7 +4,7 @@ import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.AbstractHandler;
import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.UnexpectedPostbackException;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Util;
@ -19,9 +19,9 @@ import java.io.IOException;
*/
public class PostbackHandler extends AbstractHandler {
private final BridgeAPI bridgeAPI;
private final Bridge bridgeAPI;
public PostbackHandler(BridgeAPI bridgeAPI) {
public PostbackHandler(Bridge bridgeAPI) {
this.bridgeAPI = bridgeAPI;
}

23
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/base/ForbiddenException.java
View file

@ -1,7 +1,28 @@
package uk.ac.ic.wlgitbridge.snapshot.base;
import com.google.gson.JsonElement;
import uk.ac.ic.wlgitbridge.git.exception.SnapshotAPIException;
import java.util.Arrays;
import java.util.List;
/**
* Created by winston on 25/10/15.
*/
public class ForbiddenException extends Throwable {
public class ForbiddenException extends SnapshotAPIException {
@Override
public void fromJSON(JsonElement json) {
}
@Override
public String getMessage() {
return "forbidden";
}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(getMessage());
}
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getdoc/GetDocResult.java
View file

@ -2,14 +2,14 @@ package uk.ac.ic.wlgitbridge.snapshot.getdoc;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.exception.SnapshotAPIException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.base.Request;
import uk.ac.ic.wlgitbridge.snapshot.base.Result;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.WLUser;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import uk.ac.ic.wlgitbridge.snapshot.base.Request;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.ProtectedProjectException;
/**
* Created by Winston on 06/11/14.
@ -21,7 +21,7 @@ public class GetDocResult extends Result {
private String createdAt;
private WLUser user;
private SnapshotPostException exception;
private SnapshotAPIException exception;
private ForbiddenException forbidden;
public GetDocResult(Request request, JsonElement json) throws FailedConnectionException {
@ -94,7 +94,7 @@ public class GetDocResult extends Result {
}
}
public int getVersionID() throws SnapshotPostException {
public int getVersionID() throws GitUserException {
if (exception != null) {
throw exception;
}

12
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getdoc/exception/InvalidProjectException.java
View file

@ -2,8 +2,7 @@ package uk.ac.ic.wlgitbridge.snapshot.getdoc.exception;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import uk.ac.ic.wlgitbridge.git.exception.SnapshotAPIException;
import java.util.LinkedList;
import java.util.List;
@ -11,14 +10,10 @@ import java.util.List;
/**
* Created by Winston on 08/11/14.
*/
public class InvalidProjectException extends SnapshotPostException {
public class InvalidProjectException extends SnapshotAPIException {
private List<String> errors;
public InvalidProjectException(JsonObject json) {
super(json);
}
public InvalidProjectException() {
super();
errors = new LinkedList<String>();
@ -37,7 +32,8 @@ public class InvalidProjectException extends SnapshotPostException {
@Override
public void fromJSON(JsonElement json) {
errors = new LinkedList<String>();
JsonArray errors = json.getAsJsonObject().get("errors").getAsJsonArray();
JsonArray errors =
json.getAsJsonObject().get("errors").getAsJsonArray();
for (JsonElement error : errors) {
this.errors.add(error.getAsString());
}

9
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/push/exception/SnapshotPostException.java
View file

@ -1,14 +1,12 @@
package uk.ac.ic.wlgitbridge.snapshot.push.exception;
import com.google.gson.JsonElement;
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
import java.util.List;
import uk.ac.ic.wlgitbridge.git.exception.SnapshotAPIException;
/**
* Created by Winston on 16/11/14.
*/
public abstract class SnapshotPostException extends Exception implements JSONSource {
public abstract class SnapshotPostException extends SnapshotAPIException {
public SnapshotPostException() {
@ -18,7 +16,4 @@ public abstract class SnapshotPostException extends Exception implements JSONSou
fromJSON(jsonElement);
}
public abstract String getMessage();
public abstract List<String> getDescriptionLines();
}

9
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java
View file

@ -19,6 +19,15 @@ public class Util {
private static String POSTBACK_URL;
private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS");
public static TimerTask makeTimerTask(Runnable lamb) {
return new TimerTask() {
@Override
public void run() {
lamb.run();
}
};
}
public static String entries(int entries) {
if (entries == 1) {
return "entry";

58
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java Normal file
View file

@ -0,0 +1,58 @@
package uk.ac.ic.wlgitbridge.bridge;
import org.junit.Before;
import org.junit.Test;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
/**
* Created by winston on 20/08/2016.
*/
public class BridgeTest {
private Bridge bridge;
private ProjectLock lock;
private RepoStore repoStore;
private DBStore dbStore;
private SwapStore swapStore;
private SnapshotAPI snapshotAPI;
private ResourceCache resourceCache;
private SwapJob swapJob;
@Before
public void setup() {
lock = mock(ProjectLock.class);
repoStore = mock(RepoStore.class);
dbStore = mock(DBStore.class);
swapStore = mock(SwapStore.class);
snapshotAPI = mock(SnapshotAPI.class);
swapJob = mock(SwapJob.class);
bridge = new Bridge(
lock,
repoStore,
dbStore,
swapStore,
snapshotAPI,
resourceCache,
swapJob
);
}
@Test
public void shutdownStopsSwapJob() {
bridge.startSwapJob(1000);
bridge.doShutdown();
verify(swapJob).start(1000);
verify(swapJob).stop();
}
}

18
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java
View file

@ -9,10 +9,12 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockserver.client.server.MockServerClient;
import org.mockserver.junit.MockServerRule;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import java.io.IOException;
import java.util.HashMap;
@ -32,7 +34,7 @@ public class ResourceFetcherTest {
private MockServerClient mockServerClient;
@Test
public void fetchesFilesThatAreMissingFromUrlStoreCache() throws IOException, SnapshotPostException {
public void fetchesFilesThatAreMissingFromUrlStoreCache() throws IOException, GitUserException {
final String testProjectName = "123abc";
final String testUrl = "http://localhost:" + mockServerRule.getPort() + "/123abc";
final String oldTestPath = "testPath";
@ -50,23 +52,23 @@ public class ResourceFetcherTest {
);
final Mockery context = new Mockery();
final PersistentStore persistentStore = context.mock(PersistentStore.class);
final DBStore dbStore = context.mock(DBStore.class);
context.checking(new Expectations() {{
// It should fetch the file once it finds it is missing.
oneOf(persistentStore).getPathForURLInProject(testProjectName, testUrl);
oneOf(dbStore).getPathForURLInProject(testProjectName, testUrl);
will(returnValue(oldTestPath));
// It should update the URL index store once it has fetched; at present, it does not actually change the stored path.
oneOf(persistentStore).addURLIndexForProject(testProjectName, testUrl, oldTestPath);
oneOf(dbStore).addURLIndexForProject(testProjectName, testUrl, oldTestPath);
}});
ResourceFetcher resourceFetcher = new ResourceFetcher(persistentStore);
ResourceCache resources = new UrlResourceCache(dbStore);
TemporaryFolder repositoryFolder = new TemporaryFolder();
repositoryFolder.create();
Repository repository = new FileRepositoryBuilder().setWorkTree(repositoryFolder.getRoot()).build();
Map<String, RawFile> fileTable = new RepositoryObjectTreeWalker(repository).getDirectoryContents().getFileTable();
Map<String, byte[]> fetchedUrls = new HashMap<String, byte[]>();
resourceFetcher.get(testProjectName, testUrl, newTestPath, fileTable, fetchedUrls);
resources.get(testProjectName, testUrl, newTestPath, fileTable, fetchedUrls);
// We don't bother caching in this case, at present.
assertEquals(0, fetchedUrls.size());

4
services/git-bridge/writelatex-git-bridge.iml
View file

@ -13,7 +13,7 @@
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.jmock:jmock-junit4:2.8.2" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.jmock:jmock:2.8.2" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.jmock:jmock-testjar:2.8.2" level="project" />
@ -107,5 +107,7 @@
<orderEntry type="library" scope="TEST" name="Maven: org.bouncycastle:bcprov-jdk15on:1.52" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: janino:janino:2.5.10" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.mock-server:mockserver-logging:3.10.4" level="project" />
<orderEntry type="library" name="Maven: org.mockito:mockito-core:1.10.19" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.objenesis:objenesis:2.1" level="project" />
</component>
</module>