Try to download file if it's not in the git tree

services/git-bridge/pom.xml

@@ -102,6 +102,11 @@
             <artifactId>joda-time</artifactId>
             <version>2.7</version>
         </dependency>
+        <dependency>
+            <groupId>org.mock-server</groupId>
+            <artifactId>mockserver-netty</artifactId>
+            <version>3.10.1</version>
+        </dependency>
     </dependencies>
 
 </project>

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/DataStore.java

@@ -10,6 +10,7 @@ import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
 import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
 import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
 import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
+import uk.ac.ic.wlgitbridge.data.model.db.SqlitePersistentStore;
 import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
 import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
@@ -31,7 +32,7 @@ public class DataStore {
 
     public DataStore(String rootGitDirectoryPath) {
         rootGitDirectory = initRootGitDirectory(rootGitDirectoryPath);
-        persistentStore = new PersistentStore(rootGitDirectory);
+        persistentStore = new SqlitePersistentStore(rootGitDirectory);
         List<String> excludedFromDeletion = persistentStore.getProjectNames();
         excludedFromDeletion.add(".wlgb");
         Util.deleteInDirectoryApartFrom(rootGitDirectory, excludedFromDeletion.toArray(new String[] {}));

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcher.java

@@ -42,9 +42,15 @@ public class ResourceFetcher {
             if (contents == null) {
                 RawFile rawFile = new RepositoryObjectTreeWalker(repository).getDirectoryContents().getFileTable().get(path);
                 if (rawFile == null) {
-                    throw new IllegalStateException("file was not in the current commit, or the git tree, yet path was not null");
+                    Util.sout(
+                        "WARNING: " +
+                        "File " + path + " was not in the current commit, or the git tree, yet path was not null. " +
+                        "File url is: " + url
+                    );
+                    contents = fetch(projectName, url, path);
+                } else {
+                    contents = rawFile.getContents();
                 }
-                contents = rawFile.getContents();
             }
         }
         return new RepositoryFile(newPath, contents);

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/PersistentStore.java

@@ -1,72 +1,20 @@
 package uk.ac.ic.wlgitbridge.data.model.db;
 
-import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLiteWLDatabase;
-
-import java.io.File;
-import java.sql.SQLException;
 import java.util.List;
 
 /**
- * Created by Winston on 19/11/14.
+ * Created by m on 20/11/15.
  */
-public class PersistentStore {
+public interface PersistentStore {
+    List<String> getProjectNames();
 
-    private final SQLiteWLDatabase database;
+    void setLatestVersionForProject(String project, int versionID);
 
-    public PersistentStore(File rootGitDirectory) {
+    int getLatestVersionForProject(String project);
-        try {
-            database = new SQLiteWLDatabase(rootGitDirectory);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        } catch (ClassNotFoundException e) {
-            throw new RuntimeException(e);
-        }
-    }
 
-    public List<String> getProjectNames() {
+    void addURLIndexForProject(String projectName, String url, String path);
-        try {
-            return database.getProjectNames();
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
-    public void setLatestVersionForProject(String project, int versionID) {
-        try {
-            database.setVersionIDForProject(project, versionID);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
 
-    public int getLatestVersionForProject(String project) {
+    void deleteFilesForProject(String project, String... files);
-        try {
-            return database.getVersionIDForProjectName(project);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public void addURLIndexForProject(String projectName, String url, String path) {
-        try {
-            database.addURLIndex(projectName, url, path);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public void deleteFilesForProject(String project, String... files) {
-        try {
-            database.deleteFilesForProject(project, files);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
-    public String getPathForURLInProject(String projectName, String url) {
-        try {
-            return database.getPathForURLInProject(projectName, url);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
 
+    String getPathForURLInProject(String projectName, String url);
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/SqlitePersistentStore.java

@@ -0,0 +1,78 @@
+package uk.ac.ic.wlgitbridge.data.model.db;
+
+import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLiteWLDatabase;
+
+import java.io.File;
+import java.sql.SQLException;
+import java.util.List;
+
+/**
+ * Created by Winston on 19/11/14.
+ */
+public class SqlitePersistentStore implements PersistentStore {
+
+    private final SQLiteWLDatabase database;
+
+    public SqlitePersistentStore(File rootGitDirectory) {
+        try {
+            database = new SQLiteWLDatabase(rootGitDirectory);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        } catch (ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public List<String> getProjectNames() {
+        try {
+            return database.getProjectNames();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+    @Override
+    public void setLatestVersionForProject(String project, int versionID) {
+        try {
+            database.setVersionIDForProject(project, versionID);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public int getLatestVersionForProject(String project) {
+        try {
+            return database.getVersionIDForProjectName(project);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void addURLIndexForProject(String projectName, String url, String path) {
+        try {
+            database.addURLIndex(projectName, url, path);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void deleteFilesForProject(String project, String... files) {
+        try {
+            database.deleteFilesForProject(project, files);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+    @Override
+    public String getPathForURLInProject(String projectName, String url) {
+        try {
+            return database.getPathForURLInProject(projectName, url);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+}

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java

@@ -0,0 +1,91 @@
+package uk.ac.ic.wlgitbridge.data.model;
+
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.mockserver.model.HttpRequest.request;
+import static org.mockserver.model.HttpResponse.response;
+
+import org.eclipse.jgit.lib.*;
+import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockserver.client.server.MockServerClient;
+import org.mockserver.junit.MockServerRule;
+import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
+
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by m on 20/11/15.
+ */
+public class ResourceFetcherTest {
+    @Rule
+    public MockServerRule mockServerRule = new MockServerRule(this);
+
+    private MockServerClient mockServerClient;
+
+    @Test
+    public void fetchesFilesThatAreMissingFromUrlStoreCache() throws IOException {
+        final String testProjectName = "123abc";
+        final String testUrl = "http://localhost:" + mockServerRule.getHttpPort() + "/123abc";
+
+        mockServerClient.when(
+            request()
+            .withMethod("GET")
+            .withPath("/123abc")
+        )
+        .respond(
+            response()
+            .withStatusCode(200)
+            .withBody("content")
+        );
+
+        final PersistentStore persistentStore = new PersistentStore() {
+            @Override
+            public List<String> getProjectNames() {
+                return null;
+            }
+
+            @Override
+            public void setLatestVersionForProject(String project, int versionID) {
+
+            }
+
+            @Override
+            public int getLatestVersionForProject(String project) {
+                return 0;
+            }
+
+            @Override
+            public void addURLIndexForProject(String projectName, String url, String path) {
+
+            }
+
+            @Override
+            public void deleteFilesForProject(String project, String... files) {
+
+            }
+
+            @Override
+            public String getPathForURLInProject(String projectName, String url) {
+                assertEquals(testProjectName, projectName);
+                assertEquals(testUrl, url);
+                return "missingPath";
+            }
+        };
+
+        ResourceFetcher resourceFetcher = new ResourceFetcher(persistentStore);
+        TemporaryFolder repositoryFolder = new TemporaryFolder();
+        repositoryFolder.create();
+        Repository repository = new FileRepositoryBuilder().setWorkTree(repositoryFolder.getRoot()).build();
+        Map<String, byte[]> fetchedUrls = new HashMap<String, byte[]>();
+        resourceFetcher.get(testProjectName, testUrl, "testPath", repository, fetchedUrls);
+                //     public RawFile get(String projectName, String url, String newPath, Repository repository, Map<String, byte[]> fetchedUrls) throws IOException {
+
+    }
+}

services/git-bridge/writelatex-git-bridge.iml

@@ -44,5 +44,43 @@
     <orderEntry type="library" name="Maven: org.xerial:sqlite-jdbc:3.8.7" level="project" />
     <orderEntry type="library" name="Maven: org.slf4j:slf4j-nop:1.7.9" level="project" />
     <orderEntry type="library" name="Maven: joda-time:joda-time:2.7" level="project" />
+    <orderEntry type="library" name="Maven: org.mock-server:mockserver-netty:3.10.1" level="project" />
+    <orderEntry type="library" name="Maven: org.mock-server:mockserver-client-java:3.10.1" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.4" level="project" />
+    <orderEntry type="library" name="Maven: org.mock-server:mockserver-core:3.10.1" level="project" />
+    <orderEntry type="library" name="Maven: io.netty:netty-codec-socks:4.0.31.Final" level="project" />
+    <orderEntry type="library" name="Maven: com.jcraft:jzlib:1.1.3" level="project" />
+    <orderEntry type="library" name="Maven: org.bouncycastle:bcmail-jdk15on:1.52" level="project" />
+    <orderEntry type="library" name="Maven: org.bouncycastle:bcpkix-jdk15on:1.52" level="project" />
+    <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.5.3" level="project" />
+    <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.5.3" level="project" />
+    <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.5.3" level="project" />
+    <orderEntry type="library" name="Maven: org.skyscreamer:jsonassert:1.2.3" level="project" />
+    <orderEntry type="library" name="Maven: org.json:json:20090211" level="project" />
+    <orderEntry type="library" name="Maven: com.github.fge:json-schema-validator:2.2.6" level="project" />
+    <orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:3.0.0" level="project" />
+    <orderEntry type="library" name="Maven: com.googlecode.libphonenumber:libphonenumber:6.2" level="project" />
+    <orderEntry type="library" name="Maven: com.github.fge:json-schema-core:1.2.5" level="project" />
+    <orderEntry type="library" name="Maven: com.github.fge:uri-template:0.9" level="project" />
+    <orderEntry type="library" name="Maven: com.github.fge:msg-simple:1.1" level="project" />
+    <orderEntry type="library" name="Maven: com.github.fge:btf:1.2" level="project" />
+    <orderEntry type="library" name="Maven: com.github.fge:jackson-coreutils:1.8" level="project" />
+    <orderEntry type="library" name="Maven: org.mozilla:rhino:1.7R4" level="project" />
+    <orderEntry type="library" name="Maven: javax.mail:mailapi:1.4.3" level="project" />
+    <orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
+    <orderEntry type="library" name="Maven: net.sf.jopt-simple:jopt-simple:4.6" level="project" />
+    <orderEntry type="library" name="Maven: commons-io:commons-io:1.3.2" level="project" />
+    <orderEntry type="library" name="Maven: io.netty:netty-buffer:4.0.31.Final" level="project" />
+    <orderEntry type="library" name="Maven: io.netty:netty-codec:4.0.31.Final" level="project" />
+    <orderEntry type="library" name="Maven: io.netty:netty-codec-http:4.0.31.Final" level="project" />
+    <orderEntry type="library" name="Maven: io.netty:netty-common:4.0.31.Final" level="project" />
+    <orderEntry type="library" name="Maven: io.netty:netty-handler:4.0.31.Final" level="project" />
+    <orderEntry type="library" name="Maven: io.netty:netty-transport:4.0.31.Final" level="project" />
+    <orderEntry type="library" name="Maven: org.bouncycastle:bcprov-jdk15on:1.52" level="project" />
+    <orderEntry type="library" name="Maven: com.google.guava:guava:18.0" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: ch.qos.logback:logback-classic:1.1.3" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: ch.qos.logback:logback-core:1.1.3" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: janino:janino:2.5.10" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: org.mock-server:mockserver-logging:3.10.1" level="project" />
   </component>
 </module>