github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-07 20:31:06 -05:00
Code Issues Projects Releases Packages Wiki Activity

Finished writing persistent store.

Browse source
This commit is contained in:
Winston Li 2014-11-19 20:57:31 +00:00
parent dc47c17264
commit 832009e6cf
18 changed files with 211 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
services/git-bridge/src/uk/ac/ic/wlgitbridge/application/WLGitBridgeServer.java
View file

@ -51,6 +51,7 @@ public class WLGitBridgeServer {
public void start() {
try {
jettyServer.start();
System.out.println();
System.out.println("WriteLatex-Git Bridge server started");
System.out.println("Listening on port: " + port);
System.out.println("Root git directory path: " + rootGitDirectoryPath);

16
services/git-bridge/src/uk/ac/ic/wlgitbridge/util/Util.java Normal file
View file

@ -0,0 +1,16 @@
package uk.ac.ic.wlgitbridge.util;
/**
* Created by Winston on 19/11/14.
*/
public class Util {
public static String entries(int entries) {
if (entries == 1) {
return "entry";
} else {
return "entries";
}
}
}

5
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/SnapshotFetcher.java
View file

@ -37,7 +37,7 @@ public class SnapshotFetcher implements PersistentStoreSource {
for (Snapshot snapshot : newSnapshots) {
persistentStore.addSnapshot(projectName, snapshot.getVersionID());
}
System.out.println("Snapshots fetched: " + newSnapshots);
System.out.println("Fetched snapshots: " + newSnapshots);
return newSnapshots;
}
@ -148,4 +148,7 @@ public class SnapshotFetcher implements PersistentStoreSource {
newSnapshots.add(snapshot);
}
public SortedSet<Integer> getVersions() {
return versions;
}
}

4
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/filestore/blob/Blob.java
View file

@ -1,11 +1,13 @@
package uk.ac.ic.wlgitbridge.writelatex.filestore.blob;
import uk.ac.ic.wlgitbridge.writelatex.api.request.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.writelatex.filestore.node.AttachmentNode;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreUpdater;
/**
* Created by Winston on 14/11/14.
*/
public abstract class Blob {
public abstract class Blob implements PersistentStoreUpdater<AttachmentNode> {
public abstract byte[] getContents() throws FailedConnectionException;

8
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/filestore/blob/ByteBlob.java
View file

@ -1,5 +1,8 @@
package uk.ac.ic.wlgitbridge.writelatex.filestore.blob;
import uk.ac.ic.wlgitbridge.writelatex.filestore.node.AttachmentNode;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreAPI;
/**
* Created by Winston on 14/11/14.
*/
@ -16,4 +19,9 @@ public class ByteBlob extends Blob {
return contents;
}
@Override
public void updatePersistentStore(PersistentStoreAPI persistentStore, AttachmentNode node) {
persistentStore.addFileNodeBlob(node.getProjectName(), node.getFilePath(), node.isChanged(), contents);
}
}

7
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/filestore/blob/ExternalBlob.java
View file

@ -5,6 +5,8 @@ import com.ning.http.client.AsyncHttpClient;
import com.ning.http.client.HttpResponseBodyPart;
import com.ning.http.client.Response;
import uk.ac.ic.wlgitbridge.writelatex.api.request.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.writelatex.filestore.node.AttachmentNode;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreAPI;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@ -58,4 +60,9 @@ public class ExternalBlob extends Blob {
}
}
@Override
public void updatePersistentStore(PersistentStoreAPI persistentStore, AttachmentNode node) {
persistentStore.addFileNodeExternal(node.getProjectName(), node.getFilePath(), node.isChanged(), node.getURL());
}
}

13
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/filestore/node/AttachmentNode.java
View file

@ -7,6 +7,7 @@ import uk.ac.ic.wlgitbridge.writelatex.filestore.blob.ByteBlob;
import uk.ac.ic.wlgitbridge.writelatex.filestore.store.FileIndexStore;
import uk.ac.ic.wlgitbridge.writelatex.filestore.blob.Blob;
import uk.ac.ic.wlgitbridge.writelatex.filestore.blob.ExternalBlob;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreAPI;
import java.util.Map;
@ -15,6 +16,8 @@ import java.util.Map;
*/
public class AttachmentNode extends FileNode {
private String projectName;
private final String url;
private Blob blob;
@ -45,6 +48,12 @@ public class AttachmentNode extends FileNode {
return blob;
}
@Override
public void updatePersistentStore(PersistentStoreAPI persistentStore, String projectName) {
this.projectName = projectName;
getBlob().updatePersistentStore(persistentStore, this);
}
public String getURL() {
return url;
}
@ -58,4 +67,8 @@ public class AttachmentNode extends FileNode {
}
}
public String getProjectName() {
return projectName;
}
}

11
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/filestore/node/BlobNode.java
View file

@ -6,6 +6,7 @@ import uk.ac.ic.wlgitbridge.writelatex.filestore.RepositoryFile;
import uk.ac.ic.wlgitbridge.writelatex.filestore.blob.Blob;
import uk.ac.ic.wlgitbridge.writelatex.filestore.blob.ByteBlob;
import uk.ac.ic.wlgitbridge.writelatex.filestore.blob.RawFileBlob;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreAPI;
import java.io.File;
import java.io.IOException;
@ -43,10 +44,18 @@ public class BlobNode extends FileNode {
return blob;
}
@Override
public void updatePersistentStore(PersistentStoreAPI persistentStore, String projectName) {
try {
persistentStore.addFileNodeBlob(projectName, getFilePath(), isChanged(), getBlob().getContents());
} catch (FailedConnectionException e) {
throw new RuntimeException(e);
}
}
private void writeChanged(File projectAttDirectory) throws FailedConnectionException, IOException {
if (isChanged()) {
writeToDisk(projectAttDirectory);
}
}
}

3
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/filestore/node/FileNode.java
View file

@ -3,6 +3,7 @@ package uk.ac.ic.wlgitbridge.writelatex.filestore.node;
import uk.ac.ic.wlgitbridge.bridge.RawFile;
import uk.ac.ic.wlgitbridge.writelatex.api.request.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.writelatex.filestore.blob.Blob;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreUpdater;
import java.io.File;
import java.io.FileOutputStream;
@ -13,7 +14,7 @@ import java.util.Map;
/**
* Created by Winston on 12/11/14.
*/
public abstract class FileNode {
public abstract class FileNode implements PersistentStoreUpdater<String> {
private final String filePath;
private final boolean changed;

38
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/filestore/node/WLDirectoryNode.java
View file

@ -8,6 +8,9 @@ import uk.ac.ic.wlgitbridge.writelatex.filestore.RepositoryFile;
import uk.ac.ic.wlgitbridge.writelatex.filestore.store.FileIndexStore;
import uk.ac.ic.wlgitbridge.writelatex.filestore.store.WLFileStore;
import uk.ac.ic.wlgitbridge.writelatex.model.Snapshot;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreAPI;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreSource;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreUpdater;
import java.io.File;
import java.io.IOException;
@ -20,22 +23,49 @@ import java.util.Map.Entry;
/**
* Created by Winston on 08/11/14.
*/
public class WLDirectoryNode {
public class WLDirectoryNode implements PersistentStoreSource, PersistentStoreUpdater<Void> {
private final String projectName;
private Map<String, FileNode> fileNodeTable;
private FileIndexStore fileIndexStore;
public WLDirectoryNode(String projectName) {
this(projectName, new HashMap<String, FileNode>(), new FileIndexStore());
public WLDirectoryNode(String projectName, PersistentStoreAPI persistentStore) {
this(projectName);
initFromPersistentStore(persistentStore);
}
public WLDirectoryNode(String projectName, Map<String, FileNode> fileNodeTable, FileIndexStore fileIndexStore) {
private WLDirectoryNode(String projectName) {
this.projectName = projectName;
}
private WLDirectoryNode(String projectName, Map<String, FileNode> fileNodeTable, FileIndexStore fileIndexStore) {
this.projectName = projectName;
this.fileNodeTable = fileNodeTable;
this.fileIndexStore = fileIndexStore;
}
@Override
public void initFromPersistentStore(PersistentStoreAPI persistentStore) {
fileNodeTable = new HashMap<String, FileNode>();
for (FileNode fileNode : persistentStore.getFileNodesForProjectName(projectName)) {
fileNodeTable.put(fileNode.getFilePath(), fileNode);
}
fileIndexStore = new FileIndexStore(projectName, persistentStore);
}
@Override
public void updatePersistentStore(PersistentStoreAPI persistentStore, Void info) {
updateFileNodeTableInPersistentStore(persistentStore);
fileIndexStore.updatePersistentStore(persistentStore, projectName);
}
private void updateFileNodeTableInPersistentStore(PersistentStoreAPI persistentStore) {
persistentStore.deleteFileNodesForProjectName(projectName);
for (FileNode fileNode : fileNodeTable.values()) {
fileNode.updatePersistentStore(persistentStore, projectName);
}
}
public List<FileNode> getFileNodes() {
return new LinkedList<FileNode>(fileNodeTable.values());
}

35
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/filestore/store/FileIndexStore.java
View file

@ -1,22 +1,28 @@
package uk.ac.ic.wlgitbridge.writelatex.filestore.store;
import uk.ac.ic.wlgitbridge.writelatex.api.request.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.writelatex.filestore.node.AttachmentNode;
import uk.ac.ic.wlgitbridge.writelatex.filestore.node.BlobNode;
import uk.ac.ic.wlgitbridge.writelatex.filestore.node.FileNode;
import uk.ac.ic.wlgitbridge.writelatex.filestore.node.FileNodeIndexer;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreAPI;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreSource;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreUpdater;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
/**
* Created by Winston on 08/11/14.
*/
public class FileIndexStore implements FileNodeIndexer {
public class FileIndexStore implements FileNodeIndexer, PersistentStoreSource, PersistentStoreUpdater<String> {
private final Map<BlobHash, FileNode> blobHashMappings;
private final Map<String, FileNode> urlMappings;
private Map<String, FileNode> urlMappings;
private String projectName;
public FileIndexStore(List<FileNode> fileNodes) {
blobHashMappings = new HashMap<BlobHash, FileNode>();
@ -26,8 +32,10 @@ public class FileIndexStore implements FileNodeIndexer {
}
}
public FileIndexStore() {
this(new LinkedList<FileNode>());
public FileIndexStore(String projectName, PersistentStoreAPI persistentStore) {
this.projectName = projectName;
blobHashMappings = new HashMap<BlobHash, FileNode>();
initFromPersistentStore(persistentStore);
}
@Override
@ -40,6 +48,11 @@ public class FileIndexStore implements FileNodeIndexer {
urlMappings.put(attachmentNode.getURL(), attachmentNode);
}
@Override
public void initFromPersistentStore(PersistentStoreAPI persistentStore) {
urlMappings = persistentStore.getURLIndexTableForProjectName(projectName);
}
public boolean hasAttachmentWithURL(String url) {
return urlMappings.containsKey(url);
}
@ -48,6 +61,18 @@ public class FileIndexStore implements FileNodeIndexer {
return urlMappings.get(url);
}
@Override
public void updatePersistentStore(PersistentStoreAPI persistentStore, String projectName) {
persistentStore.deleteURLIndexesForProjectName(projectName);
for (Entry<String, FileNode> urlMapping : urlMappings.entrySet()) {
try {
persistentStore.addURLIndex(projectName, urlMapping.getKey(), urlMapping.getValue().getContents());
} catch (FailedConnectionException e) {
throw new RuntimeException(e);
}
}
}
}
/*Winston is really cool

20
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/filestore/store/WLFileStore.java
View file

@ -10,6 +10,7 @@ import uk.ac.ic.wlgitbridge.writelatex.filestore.node.WLDirectoryNode;
import uk.ac.ic.wlgitbridge.writelatex.model.Snapshot;
import uk.ac.ic.wlgitbridge.writelatex.model.WLProject;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreAPI;
import uk.ac.ic.wlgitbridge.writelatex.model.db.PersistentStoreSource;
import java.io.File;
import java.io.IOException;
@ -18,12 +19,14 @@ import java.util.*;
/**
* Created by Winston on 08/11/14.
*/
public class WLFileStore {
public class WLFileStore implements PersistentStoreSource {
private final Map<String, WLDirectoryNode> fileStore;
private final File rootGitDirectory;
private final File attDirectory;
private PersistentStoreAPI persistentStore;
public WLFileStore(File rootGitDirectory) {
fileStore = new HashMap<String, WLDirectoryNode>();
this.rootGitDirectory = rootGitDirectory;
@ -33,7 +36,15 @@ public class WLFileStore {
public WLFileStore(File rootGitDirectory, PersistentStoreAPI persistentStoreAPI) {
this(rootGitDirectory);
initFromPersistentStore(persistentStoreAPI);
}
@Override
public void initFromPersistentStore(PersistentStoreAPI persistentStore) {
this.persistentStore = persistentStore;
for (String projectName : persistentStore.getProjectNames()) {
fileStore.put(projectName, new WLDirectoryNode(projectName, persistentStore));
}
}
public static void deleteInDirectory(File directory) {
@ -64,6 +75,7 @@ public class WLFileStore {
projectName,
snapshot));
}
directoryNode.updatePersistentStore(persistentStore, null);
return writableRepositories;
}
@ -72,13 +84,15 @@ public class WLFileStore {
}
public void approveCandidateSnapshot(CandidateSnapshot candidateSnapshot) {
fileStore.put(candidateSnapshot.getProjectName(), candidateSnapshot.getDirectoryNode());
WLDirectoryNode directoryNode = candidateSnapshot.getDirectoryNode();
fileStore.put(candidateSnapshot.getProjectName(), directoryNode);
directoryNode.updatePersistentStore(persistentStore, null);
}
private WLDirectoryNode getDirectoryNodeForProjectName(String projectName) {
WLDirectoryNode directoryNode = fileStore.get(projectName);
if (directoryNode == null) {
directoryNode = new WLDirectoryNode(projectName);
directoryNode = new WLDirectoryNode(projectName, persistentStore);
fileStore.put(projectName, directoryNode);
}
return directoryNode;

3
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/model/WLDataModel.java
View file

@ -29,10 +29,11 @@ public class WLDataModel implements CandidateSnapshotCallback {
File rootGitDirectory = initRootGitDirectory(rootGitDirectoryPath);
persistentStore = new WLGBPersistentStore(rootGitDirectory);
projectStore = persistentStore.loadProjectStore();
System.out.println("Loaded projects: " + projectStore.getProjectNames().size() + ".");
fileStore = persistentStore.loadFileStore();
System.out.println("Loaded file store and index tables.");
List<String> excludedFromDeletion = projectStore.getProjectNames();
excludedFromDeletion.add(".wlgb");
System.out.println("Not deleting: " + excludedFromDeletion);
WLFileStore.deleteInDirectoryApartFrom(rootGitDirectory, excludedFromDeletion.toArray(new String[]{}));
}

3
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/model/WLProject.java
View file

@ -15,6 +15,7 @@ import java.util.SortedSet;
*/
public class WLProject implements PersistentStoreSource {
public static final int INVALID_VERSION_ID = -1;
private final String name;
private final Map<Integer, Snapshot> snapshots;
private final SnapshotFetcher snapshotFetcher;
@ -49,7 +50,7 @@ public class WLProject implements PersistentStoreSource {
private void updateLatestSnapshot() {
Snapshot latest = snapshotFetcher.getLatestSnapshot();
if (latest == null) {
latestSnapshotID = -1;
latestSnapshotID = INVALID_VERSION_ID;
} else {
latestSnapshotID = latest.getVersionID();
}

7
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/model/db/PersistentStoreAPI.java
View file

@ -17,8 +17,8 @@ public interface PersistentStoreAPI {
public void addProject(String name);
public void addSnapshot(String projectName, int versionID);
public void addFileNodeBlob(String projectName, String fileName, int changed, byte[] blob);
public void addFileNodeExternal(String projectName, String fileName, int changed, String url);
public void addFileNodeBlob(String projectName, String fileName, boolean changed, byte[] blob);
public void addFileNodeExternal(String projectName, String fileName, boolean changed, String url);
public void addURLIndex(String projectName, String url, byte[] blob);
public List<String> getProjectNames();
@ -26,4 +26,7 @@ public interface PersistentStoreAPI {
public List<FileNode> getFileNodesForProjectName(String projectName);
public Map<String, FileNode> getURLIndexTableForProjectName(String projectName);
public void deleteFileNodesForProjectName(String projectName);
public void deleteURLIndexesForProjectName(String projectName);
}

10
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/model/db/PersistentStoreUpdater.java Normal file
View file

@ -0,0 +1,10 @@
package uk.ac.ic.wlgitbridge.writelatex.model.db;
/**
* Created by Winston on 19/11/14.
*/
public interface PersistentStoreUpdater<T> {
public void updatePersistentStore(PersistentStoreAPI persistentStore, T info);
}

27
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/model/db/SQLiteWLDatabase.java
View file

@ -66,6 +66,11 @@ public class SQLiteWLDatabase {
private static final String GET_URL_INDEXES_FOR_PROJECT_NAME =
"SELECT `url`, `blob` FROM `url_index_store` WHERE `project_name` = ?";
private static final String DELETE_FILE_NODES_FOR_PROJECT_NAME =
"DELETE FROM `file_node_table` WHERE `project_name` = ?";
private static final String DELETE_URL_INDEXES_FOR_PROJECT_NAME =
"DELETE FROM `url_index_store` WHERE `project_name` = ?";
private final File rootGitDirectory;
private final Connection connection;
@ -80,6 +85,9 @@ public class SQLiteWLDatabase {
private PreparedStatement getFileNodesForProjectNameStatement;
private PreparedStatement getURLIndexesForProjectNameStatement;
private PreparedStatement deleteFileNodesForProjectNameStatement;
private PreparedStatement deleteURLIndexesForProjectNameStatement;
public SQLiteWLDatabase(File rootGitDirectory) throws SQLException, ClassNotFoundException {
this.rootGitDirectory = rootGitDirectory;
File databaseFile = new File(rootGitDirectory, "/.wlgb/wlgb.db");
@ -108,6 +116,9 @@ public class SQLiteWLDatabase {
getVersionIDsForProjectNameStatement = connection.prepareStatement(GET_VERSION_IDS_FOR_PROJECT_NAME);
getFileNodesForProjectNameStatement = connection.prepareStatement(GET_FILE_NODES_FOR_PROJECT_NAME);
getURLIndexesForProjectNameStatement = connection.prepareStatement(GET_URL_INDEXES_FOR_PROJECT_NAME);
deleteFileNodesForProjectNameStatement = connection.prepareStatement(DELETE_FILE_NODES_FOR_PROJECT_NAME);
deleteURLIndexesForProjectNameStatement = connection.prepareStatement(DELETE_URL_INDEXES_FOR_PROJECT_NAME);
}
public void addProject(String name) throws SQLException {
@ -202,6 +213,18 @@ public class SQLiteWLDatabase {
return urlIndexTable;
}
public void deleteFileNodesForProjectName(String projectName) throws SQLException {
deleteFileNodesForProjectNameStatement.clearParameters();
deleteFileNodesForProjectNameStatement.setString(1, projectName);
deleteFileNodesForProjectNameStatement.executeUpdate();
}
public void deleteURLIndexesForProjectName(String projectName) throws SQLException {
deleteURLIndexesForProjectNameStatement.clearParameters();
deleteURLIndexesForProjectNameStatement.setString(1, projectName);
deleteURLIndexesForProjectNameStatement.executeUpdate();
}
private void test() throws SQLException {
addProject("testproj12");
addSnapshot("testproj12", 0);
@ -211,7 +234,7 @@ public class SQLiteWLDatabase {
addURLIndex("testproj12", "http://someurl.com/urlname.jpg", "thebytes".getBytes());
}
private static int booleanToInt(boolean b) {
public static int booleanToInt(boolean b) {
if (b) {
return 1;
} else {
@ -219,7 +242,7 @@ public class SQLiteWLDatabase {
}
}
private static boolean intToBoolean(int i) {
public static boolean intToBoolean(int i) {
return i != 0;
}

26
services/git-bridge/src/uk/ac/ic/wlgitbridge/writelatex/model/db/WLGBPersistentStore.java
View file

@ -57,18 +57,18 @@ public class WLGBPersistentStore implements PersistentStoreAPI {
}
@Override
public void addFileNodeBlob(String projectName, String fileName, int changed, byte[] blob) {
public void addFileNodeBlob(String projectName, String fileName, boolean changed, byte[] blob) {
try {
database.addFileNodeBlob(projectName, fileName, changed, blob);
database.addFileNodeBlob(projectName, fileName, SQLiteWLDatabase.booleanToInt(changed), blob);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
@Override
public void addFileNodeExternal(String projectName, String fileName, int changed, String url) {
public void addFileNodeExternal(String projectName, String fileName, boolean changed, String url) {
try {
database.addFileNodeExternal(projectName, fileName, changed, url);
database.addFileNodeExternal(projectName, fileName, SQLiteWLDatabase.booleanToInt(changed), url);
} catch (SQLException e) {
throw new RuntimeException(e);
}
@ -119,4 +119,22 @@ public class WLGBPersistentStore implements PersistentStoreAPI {
}
}
@Override
public void deleteFileNodesForProjectName(String projectName) {
try {
database.deleteFileNodesForProjectName(projectName);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
@Override
public void deleteURLIndexesForProjectName(String projectName) {
try {
database.deleteURLIndexesForProjectName(projectName);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
}