From bad0d1bbcfa9302ba16cb2edeeee4d450667b211 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 18 Feb 2015 14:58:27 +0100 Subject: [PATCH 01/31] implementing github trees as described https://developer.github.com/v3/git/trees/#get-a-tree-recursively --- .../java/org/kohsuke/github/GHRepository.java | 29 +++++ src/main/java/org/kohsuke/github/GHTree.java | 114 ++++++++++++++++++ src/test/java/org/kohsuke/github/AppTest.java | 29 +++++ 3 files changed, 172 insertions(+) create mode 100644 src/main/java/org/kohsuke/github/GHTree.java diff --git a/src/main/java/org/kohsuke/github/GHRepository.java b/src/main/java/org/kohsuke/github/GHRepository.java index 8570a09bb1..1b84d25ed5 100644 --- a/src/main/java/org/kohsuke/github/GHRepository.java +++ b/src/main/java/org/kohsuke/github/GHRepository.java @@ -641,6 +641,35 @@ public GHRef getRef(String refName) throws IOException { return root.retrieve().to(String.format("/repos/%s/%s/git/refs/%s", owner.login, name, refName), GHRef.class).wrap(root); } /** + * Retrive a tree of the given type for the current GitHub repository. + * + * @param sha - sha number or branch name ex: "master" + * @return refs matching the request type + * @throws IOException + * on failure communicating with GitHub, potentially due to an + * invalid tree type being requested + */ + public GHTree getTree(String sha) throws IOException { + String url = String.format("/repos/%s/%s/git/trees/%s", owner.login, name, sha); + return root.retrieve().to(url, GHTree.class).wrap(root); + } + + /** + * Retrieves the tree for the current GitHub repository, recursively as described in here: + * https://developer.github.com/v3/git/trees/#get-a-tree-recursively + * + * @param sha - sha number or branch name ex: "master" + * @param recursive use 1 + * @throws IOException + * on failure communicating with GitHub, potentially due to an + * invalid tree type being requested + */ + public GHTree getTreeRecursive(String sha, int recursive) throws IOException { + String url = String.format("/repos/%s/%s/git/trees/%s?recursive=%d", owner.login, name, sha, recursive); + return root.retrieve().to(url, GHTree.class).wrap(root); + } + + /** * Gets a commit object in this repository. */ public GHCommit getCommit(String sha1) throws IOException { diff --git a/src/main/java/org/kohsuke/github/GHTree.java b/src/main/java/org/kohsuke/github/GHTree.java new file mode 100644 index 0000000000..f520c14cd5 --- /dev/null +++ b/src/main/java/org/kohsuke/github/GHTree.java @@ -0,0 +1,114 @@ +package org.kohsuke.github; + +import java.net.URL; + +/** + * Provides information for Git Trees + * https://developer.github.com/v3/git/trees/ + * + * @author Daniel Teixeira - https://github.com/ddtxra + */ +public class GHTree { + /* package almost final */GitHub root; + + private boolean truncated; + private String sha, url; + private GHTreeEntry[] tree; + + /** + * The SHA for this trees + */ + public String getSha() { + return sha; + } + + /** + * Return an array of entries of the trees + * @return + */ + public GHTreeEntry[] getTree() { + return tree; + } + + /** + * Returns true if the number of items in the tree array exceeded the GitHub maximum limit. + * @return true true if the number of items in the tree array exceeded the GitHub maximum limit otherwise false. + */ + public boolean isTruncated() { + return truncated; + } + + /** + * The API URL of this tag, such as + * "url": "https://api.github.com/repos/octocat/Hello-World/trees/fc6274d15fa3ae2ab983129fb037999f264ba9a7", + */ + public URL getUrl() { + return GitHub.parseURL(url); + } + + /* package */GHTree wrap(GitHub root) { + this.root = root; + return this; + } + + public static class GHTreeEntry { + private String path, mode, type, sha, url; + private long size; + + /** + * Get the path such as + * "subdir/file.txt" + * + * @return the path + */ + public String getPath() { + return path; + } + + /** + * Get mode such as + * 100644 + * + * @return the mode + */ + public String getMode() { + return mode; + } + + /** + * Gets the size of the file, such as + * 132 + * @return The size of the path or 0 if it is a directory + */ + public long getSize() { + return size; + } + + /** + * Gets the type such as: + * "blob" + * + * @return The type + */ + public String getType() { + return type; + } + + + /** + * SHA1 of this object. + */ + public String getSha() { + return sha; + } + + /** + * API URL to this Git data, such as + * https://api.github.com/repos/jenkinsci + * /jenkins/git/commits/b72322675eb0114363a9a86e9ad5a170d1d07ac0 + */ + public URL getUrl() { + return GitHub.parseURL(url); + } + } +} diff --git a/src/test/java/org/kohsuke/github/AppTest.java b/src/test/java/org/kohsuke/github/AppTest.java index a13f5423f5..4666f2920d 100755 --- a/src/test/java/org/kohsuke/github/AppTest.java +++ b/src/test/java/org/kohsuke/github/AppTest.java @@ -3,10 +3,12 @@ import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; + import org.junit.Assume; import org.junit.Test; import org.kohsuke.github.GHCommit.File; import org.kohsuke.github.GHOrganization.Permission; +import org.kohsuke.github.GHTree.GHTreeEntry; import java.io.IOException; import java.net.URL; @@ -665,6 +667,33 @@ public void testReadme() throws IOException { assertEquals(readme.getName(),"README.md"); assertEquals(readme.getContent(),"This is a markdown readme.\n"); } + + + @Test + public void testTrees() throws IOException { + GHTree masterTree = gitHub.getRepository("kohsuke/github-api").getTree("master"); + boolean foundReadme = false; + for(GHTreeEntry e : masterTree.getTree()){ + if("readme".equalsIgnoreCase(e.getPath().replaceAll(".md", ""))){ + foundReadme = true; + break; + } + } + assertTrue(foundReadme); + } + + @Test + public void testTreesRecursive() throws IOException { + GHTree masterTree = gitHub.getRepository("kohsuke/github-api").getTreeRecursive("master", 1); + boolean foundThisFile = false; + for(GHTreeEntry e : masterTree.getTree()){ + if(e.getPath().endsWith(AppTest.class.getSimpleName() + ".java")){ + foundThisFile = true; + break; + } + } + assertTrue(foundThisFile); + } @Test public void testRepoLabel() throws IOException { From a716a59489636b5a33c4058d36af1f56847b2673 Mon Sep 17 00:00:00 2001 From: Ashwanth Kumar Date: Sun, 22 Feb 2015 09:41:58 +0530 Subject: [PATCH 02/31] Picking endpoint from the properties file and environment variables Helps seemless switching between public github and enterprise without any code changes --- src/main/java/org/kohsuke/github/GitHubBuilder.java | 13 +++++++++---- src/test/java/org/kohsuke/github/GitHubTest.java | 7 ++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GitHubBuilder.java b/src/main/java/org/kohsuke/github/GitHubBuilder.java index e794fb14ca..91c19f5165 100644 --- a/src/main/java/org/kohsuke/github/GitHubBuilder.java +++ b/src/main/java/org/kohsuke/github/GitHubBuilder.java @@ -16,9 +16,9 @@ * @since 1.59 */ public class GitHubBuilder { - private String endpoint = GitHub.GITHUB_URL; - + // default scoped so unit tests can read them. + /* private */ String endpoint = GitHub.GITHUB_URL; /* private */ String user; /* private */ String password; /* private */ String oauthToken; @@ -70,7 +70,7 @@ public static GitHubBuilder fromCredentials() throws IOException { } - public static GitHubBuilder fromEnvironment(String loginVariableName, String passwordVariableName, String oauthVariableName) throws IOException { + public static GitHubBuilder fromEnvironment(String loginVariableName, String passwordVariableName, String oauthVariableName, String endpointVariableName) throws IOException { Properties env = new Properties(); @@ -89,7 +89,11 @@ public static GitHubBuilder fromEnvironment(String loginVariableName, String pas if (oauthValue != null) env.put("oauth", oauthValue); - + + Object endPoint = System.getenv(endpointVariableName); + if (endPoint != null) + env.put("endpoint", endPoint); + return fromProperties(env); } @@ -131,6 +135,7 @@ public static GitHubBuilder fromProperties(Properties props) { GitHubBuilder self = new GitHubBuilder(); self.withOAuthToken(props.getProperty("oauth"), props.getProperty("login")); self.withPassword(props.getProperty("login"), props.getProperty("password")); + self.withEndpoint(props.getProperty("endpoint", GitHub.GITHUB_URL)); return self; } diff --git a/src/test/java/org/kohsuke/github/GitHubTest.java b/src/test/java/org/kohsuke/github/GitHubTest.java index 3e876f0b10..4ac4ecfea2 100644 --- a/src/test/java/org/kohsuke/github/GitHubTest.java +++ b/src/test/java/org/kohsuke/github/GitHubTest.java @@ -98,15 +98,16 @@ public void testGitHubBuilderFromCustomEnvironment() throws IOException { props.put("customLogin", "bogusLogin"); props.put("customOauth", "bogusOauth"); props.put("customPassword", "bogusPassword"); - + props.put("customEndpoint", "bogusEndpoint"); + setupEnvironment(props); - GitHubBuilder builder = GitHubBuilder.fromEnvironment("customLogin", "customPassword", "customOauth"); + GitHubBuilder builder = GitHubBuilder.fromEnvironment("customLogin", "customPassword", "customOauth", "customEndpoint"); assertEquals("bogusLogin", builder.user); assertEquals("bogusOauth", builder.oauthToken); assertEquals("bogusPassword", builder.password); - + assertEquals("bogusEndpoint", builder.endpoint); } } From dcc3b7f36bda43b4a4fd00f9c22e18ff0adfa183 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 2 Mar 2015 09:10:52 -0800 Subject: [PATCH 03/31] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 433cb6bdd0..ebfe7dc7b0 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ github-api - 1.63 + 1.64-SNAPSHOT GitHub API for Java http://github-api.kohsuke.org/ GitHub API for Java @@ -16,7 +16,7 @@ scm:git:git@github.com/kohsuke/${project.artifactId}.git scm:git:ssh://git@github.com/kohsuke/${project.artifactId}.git http://${project.artifactId}.kohsuke.org/ - github-api-1.63 + HEAD From 0bf81f4fb9d34633bea1e88877d67a9a2caf554e Mon Sep 17 00:00:00 2001 From: Kanstantsin Shautsou Date: Tue, 3 Mar 2015 19:23:06 +0300 Subject: [PATCH 04/31] Set credentials file according to documentation --- src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java b/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java index e45895d66e..0f3145e4fa 100644 --- a/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java +++ b/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java @@ -18,7 +18,7 @@ public abstract class AbstractGitHubApiTestBase extends Assert { @Before public void setUp() throws Exception { Properties props = new Properties(); - java.io.File f = new java.io.File(System.getProperty("user.home"), ".github.kohsuke2"); + java.io.File f = new java.io.File(System.getProperty("user.home"), ".github"); if (f.exists()) { FileInputStream in = new FileInputStream(f); try { From f78530636e375c1eefe8a7907fd5d46fece65310 Mon Sep 17 00:00:00 2001 From: khoa-nd Date: Thu, 5 Mar 2015 15:53:02 +0700 Subject: [PATCH 05/31] Add method to get the list of languages using in repository --- src/main/java/org/kohsuke/github/GHRepository.java | 12 ++++++++++++ src/test/java/org/kohsuke/github/RepositoryTest.java | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/src/main/java/org/kohsuke/github/GHRepository.java b/src/main/java/org/kohsuke/github/GHRepository.java index 8570a09bb1..a89fe11569 100644 --- a/src/main/java/org/kohsuke/github/GHRepository.java +++ b/src/main/java/org/kohsuke/github/GHRepository.java @@ -285,6 +285,18 @@ protected void wrapUp(GHTag[] page) { }; } + /** + * List languages for the specified repository. + * The value on the right of a language is the number of bytes of code written in that language. + * { + "C": 78769, + "Python": 7769 + } + */ + public Map listLanguages() throws IOException { + return root.retrieve().to(getApiTailUrl("languages"), HashMap.class); + } + public String getOwnerName() { return owner.login; } diff --git a/src/test/java/org/kohsuke/github/RepositoryTest.java b/src/test/java/org/kohsuke/github/RepositoryTest.java index 6862ace7be..61cdc09dca 100644 --- a/src/test/java/org/kohsuke/github/RepositoryTest.java +++ b/src/test/java/org/kohsuke/github/RepositoryTest.java @@ -43,4 +43,11 @@ public void listContributors() throws IOException { private GHRepository getRepository() throws IOException { return gitHub.getOrganization("github-api-test-org").getRepository("jenkins"); } + + @Test + public void listLanguages() throws IOException { + GHRepository r = gitHub.getRepository("kohsuke/github-api"); + String mainLanguage = r.getLanguage(); + assertTrue(r.listLanguages().containsKey(mainLanguage)); + } } From 374fdb37e181e8e0236636a1b4516f97b1e2dd11 Mon Sep 17 00:00:00 2001 From: khoa-nd Date: Fri, 6 Mar 2015 09:11:03 +0700 Subject: [PATCH 06/31] Change type of language bytes from Integer to Long --- src/main/java/org/kohsuke/github/GHRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/kohsuke/github/GHRepository.java b/src/main/java/org/kohsuke/github/GHRepository.java index a89fe11569..1c62c6c4d5 100644 --- a/src/main/java/org/kohsuke/github/GHRepository.java +++ b/src/main/java/org/kohsuke/github/GHRepository.java @@ -293,7 +293,7 @@ protected void wrapUp(GHTag[] page) { "Python": 7769 } */ - public Map listLanguages() throws IOException { + public Map listLanguages() throws IOException { return root.retrieve().to(getApiTailUrl("languages"), HashMap.class); } From 232c0389d31b197f7911c1e6080e3713aeb07753 Mon Sep 17 00:00:00 2001 From: Ashwanth Kumar Date: Fri, 13 Mar 2015 21:26:19 +0530 Subject: [PATCH 07/31] Adding fromEnvironment to maintain backword compatibility --- src/main/java/org/kohsuke/github/GitHubBuilder.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/kohsuke/github/GitHubBuilder.java b/src/main/java/org/kohsuke/github/GitHubBuilder.java index 91c19f5165..119a36700a 100644 --- a/src/main/java/org/kohsuke/github/GitHubBuilder.java +++ b/src/main/java/org/kohsuke/github/GitHubBuilder.java @@ -69,6 +69,10 @@ public static GitHubBuilder fromCredentials() throws IOException { } } + + public static GitHubBuilder fromEnvironment(String loginVariableName, String passwordVariableName, String oauthVariableName) throws IOException { + return fromEnvironment(loginVariableName, passwordVariableName, oauthVariableName, ""); + } public static GitHubBuilder fromEnvironment(String loginVariableName, String passwordVariableName, String oauthVariableName, String endpointVariableName) throws IOException { From 41c0dd972723caa21caf4388b7c6dcf90e4c9f79 Mon Sep 17 00:00:00 2001 From: Ashwanth Kumar Date: Fri, 13 Mar 2015 21:26:49 +0530 Subject: [PATCH 08/31] Fixing the indentation for enpointVariableName --- src/main/java/org/kohsuke/github/GitHubBuilder.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GitHubBuilder.java b/src/main/java/org/kohsuke/github/GitHubBuilder.java index 119a36700a..60828a521c 100644 --- a/src/main/java/org/kohsuke/github/GitHubBuilder.java +++ b/src/main/java/org/kohsuke/github/GitHubBuilder.java @@ -94,9 +94,10 @@ public static GitHubBuilder fromEnvironment(String loginVariableName, String pas if (oauthValue != null) env.put("oauth", oauthValue); - Object endPoint = System.getenv(endpointVariableName); - if (endPoint != null) - env.put("endpoint", endPoint); + Object endPoint = System.getenv(endpointVariableName); + + if (endPoint != null) + env.put("endpoint", endPoint); return fromProperties(env); From dfce0bda7c65973b75a9a82ac5bec2817ef14924 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 15 Mar 2015 12:48:27 -0700 Subject: [PATCH 09/31] prefer list over raw array --- src/main/java/org/kohsuke/github/GHTree.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GHTree.java b/src/main/java/org/kohsuke/github/GHTree.java index f520c14cd5..55158e9563 100644 --- a/src/main/java/org/kohsuke/github/GHTree.java +++ b/src/main/java/org/kohsuke/github/GHTree.java @@ -1,6 +1,9 @@ package org.kohsuke.github; import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; /** * Provides information for Git Trees @@ -26,8 +29,8 @@ public String getSha() { * Return an array of entries of the trees * @return */ - public GHTreeEntry[] getTree() { - return tree; + public List getTree() { + return Collections.unmodifiableList(Arrays.asList(tree)); } /** From 76610b25d792e3b5f27794cf6c1e4294038cbbc4 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 15 Mar 2015 12:49:14 -0700 Subject: [PATCH 10/31] Promoted GHTreeEntry to the top-level --- src/main/java/org/kohsuke/github/GHTree.java | 63 +--------------- .../java/org/kohsuke/github/GHTreeEntry.java | 71 +++++++++++++++++++ src/test/java/org/kohsuke/github/AppTest.java | 1 - 3 files changed, 73 insertions(+), 62 deletions(-) create mode 100644 src/main/java/org/kohsuke/github/GHTreeEntry.java diff --git a/src/main/java/org/kohsuke/github/GHTree.java b/src/main/java/org/kohsuke/github/GHTree.java index 55158e9563..90cf3c78e8 100644 --- a/src/main/java/org/kohsuke/github/GHTree.java +++ b/src/main/java/org/kohsuke/github/GHTree.java @@ -9,7 +9,8 @@ * Provides information for Git Trees * https://developer.github.com/v3/git/trees/ * - * @author Daniel Teixeira - https://github.com/ddtxra + * @author Daniel Teixeira - https://github.com/ddtxra + * @see GHRepository#getTree(String) */ public class GHTree { /* package almost final */GitHub root; @@ -54,64 +55,4 @@ public URL getUrl() { return this; } - public static class GHTreeEntry { - private String path, mode, type, sha, url; - private long size; - - /** - * Get the path such as - * "subdir/file.txt" - * - * @return the path - */ - public String getPath() { - return path; - } - - /** - * Get mode such as - * 100644 - * - * @return the mode - */ - public String getMode() { - return mode; - } - - /** - * Gets the size of the file, such as - * 132 - * @return The size of the path or 0 if it is a directory - */ - public long getSize() { - return size; - } - - /** - * Gets the type such as: - * "blob" - * - * @return The type - */ - public String getType() { - return type; - } - - - /** - * SHA1 of this object. - */ - public String getSha() { - return sha; - } - - /** - * API URL to this Git data, such as - * https://api.github.com/repos/jenkinsci - * /jenkins/git/commits/b72322675eb0114363a9a86e9ad5a170d1d07ac0 - */ - public URL getUrl() { - return GitHub.parseURL(url); - } - } } diff --git a/src/main/java/org/kohsuke/github/GHTreeEntry.java b/src/main/java/org/kohsuke/github/GHTreeEntry.java new file mode 100644 index 0000000000..e3d831c073 --- /dev/null +++ b/src/main/java/org/kohsuke/github/GHTreeEntry.java @@ -0,0 +1,71 @@ +package org.kohsuke.github; + +import java.net.URL; + +/** + * Provides information for Git Trees + * https://developer.github.com/v3/git/trees/ + * + * @author Daniel Teixeira - https://github.com/ddtxra + * @see GHTree + */ +public class GHTreeEntry { + private String path, mode, type, sha, url; + private long size; + + /** + * Get the path such as + * "subdir/file.txt" + * + * @return the path + */ + public String getPath() { + return path; + } + + /** + * Get mode such as + * 100644 + * + * @return the mode + */ + public String getMode() { + return mode; + } + + /** + * Gets the size of the file, such as + * 132 + * @return The size of the path or 0 if it is a directory + */ + public long getSize() { + return size; + } + + /** + * Gets the type such as: + * "blob" + * + * @return The type + */ + public String getType() { + return type; + } + + + /** + * SHA1 of this object. + */ + public String getSha() { + return sha; + } + + /** + * API URL to this Git data, such as + * https://api.github.com/repos/jenkinsci + * /jenkins/git/commits/b72322675eb0114363a9a86e9ad5a170d1d07ac0 + */ + public URL getUrl() { + return GitHub.parseURL(url); + } +} diff --git a/src/test/java/org/kohsuke/github/AppTest.java b/src/test/java/org/kohsuke/github/AppTest.java index 4666f2920d..36d1825882 100755 --- a/src/test/java/org/kohsuke/github/AppTest.java +++ b/src/test/java/org/kohsuke/github/AppTest.java @@ -8,7 +8,6 @@ import org.junit.Test; import org.kohsuke.github.GHCommit.File; import org.kohsuke.github.GHOrganization.Permission; -import org.kohsuke.github.GHTree.GHTreeEntry; import java.io.IOException; import java.net.URL; From 271d18cddcc3bbb16ba111a98136ef74094d93eb Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 15 Mar 2015 13:01:35 -0700 Subject: [PATCH 11/31] simplification --- .../org/kohsuke/github/GitHubBuilder.java | 37 ++++++------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GitHubBuilder.java b/src/main/java/org/kohsuke/github/GitHubBuilder.java index 60828a521c..b0b35844a0 100644 --- a/src/main/java/org/kohsuke/github/GitHubBuilder.java +++ b/src/main/java/org/kohsuke/github/GitHubBuilder.java @@ -67,40 +67,25 @@ public static GitHubBuilder fromCredentials() throws IOException { else throw new IOException("Failed to resolve credentials from ~/.github or the environment.", e); } - } public static GitHubBuilder fromEnvironment(String loginVariableName, String passwordVariableName, String oauthVariableName) throws IOException { return fromEnvironment(loginVariableName, passwordVariableName, oauthVariableName, ""); } - - public static GitHubBuilder fromEnvironment(String loginVariableName, String passwordVariableName, String oauthVariableName, String endpointVariableName) throws IOException { - - - Properties env = new Properties(); - - Object loginValue = System.getenv(loginVariableName); - - if (loginValue != null) - env.put("login", loginValue); - - Object passwordValue = System.getenv(passwordVariableName); - - if (passwordValue != null) - env.put("password", passwordValue); - - Object oauthValue = System.getenv(oauthVariableName); - - if (oauthValue != null) - env.put("oauth", oauthValue); - - Object endPoint = System.getenv(endpointVariableName); - if (endPoint != null) - env.put("endpoint", endPoint); + private static void loadIfSet(String envName, Properties p, String propName) { + String v = System.getenv(envName); + if (v != null) + p.put(propName, v); + } + public static GitHubBuilder fromEnvironment(String loginVariableName, String passwordVariableName, String oauthVariableName, String endpointVariableName) throws IOException { + Properties env = new Properties(); + loadIfSet(loginVariableName,env,"login"); + loadIfSet(passwordVariableName,env,"password"); + loadIfSet(oauthVariableName,env,"oauth"); + loadIfSet(endpointVariableName,env,"endpoint"); return fromProperties(env); - } public static GitHubBuilder fromEnvironment() throws IOException { From 690292352be918c22e895a92b29467090f995026 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 15 Mar 2015 13:02:28 -0700 Subject: [PATCH 12/31] Simplification But this method is insane! --- src/main/java/org/kohsuke/github/GitHubBuilder.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GitHubBuilder.java b/src/main/java/org/kohsuke/github/GitHubBuilder.java index b0b35844a0..24c35c4d80 100644 --- a/src/main/java/org/kohsuke/github/GitHubBuilder.java +++ b/src/main/java/org/kohsuke/github/GitHubBuilder.java @@ -89,16 +89,8 @@ public static GitHubBuilder fromEnvironment(String loginVariableName, String pas } public static GitHubBuilder fromEnvironment() throws IOException { - Properties props = new Properties(); - - Map env = System.getenv(); - - for (Map.Entry element : env.entrySet()) { - - props.put(element.getKey(), element.getValue()); - } - + props.putAll(System.getenv()); return fromProperties(props); } From 2478dad9b545128e2453c68902b4435bc5713caa Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 15 Mar 2015 13:04:50 -0700 Subject: [PATCH 13/31] Simplification --- .../org/kohsuke/github/GitHubBuilder.java | 43 ++++++++----------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GitHubBuilder.java b/src/main/java/org/kohsuke/github/GitHubBuilder.java index 24c35c4d80..f03875d122 100644 --- a/src/main/java/org/kohsuke/github/GitHubBuilder.java +++ b/src/main/java/org/kohsuke/github/GitHubBuilder.java @@ -9,7 +9,6 @@ import java.net.HttpURLConnection; import java.net.Proxy; import java.net.URL; -import java.util.Map; import java.util.Properties; /** @@ -30,43 +29,35 @@ public GitHubBuilder() { /** * First check if the credentials are configured using the ~/.github properties file. - * + * * If no user is specified it means there is no configuration present so check the environment instead. - * + * * If there is still no user it means there are no credentials defined and throw an IOException. - * + * * @return the configured Builder from credentials defined on the system or in the environment. - * + * * @throws IOException If there are no credentials defined in the ~/.github properties file or the process environment. */ public static GitHubBuilder fromCredentials() throws IOException { - + Exception cause = null; GitHubBuilder builder; + try { builder = fromPropertyFile(); - + if (builder.user != null) return builder; - else { - - // this is the case where the ~/.github file exists but has no content. - - builder = fromEnvironment(); - - if (builder.user != null) - return builder; - else - throw new IOException("Failed to resolve credentials from ~/.github or the environment."); - } - } catch (FileNotFoundException e) { - builder = fromEnvironment(); - - if (builder.user != null) - return builder; - else - throw new IOException("Failed to resolve credentials from ~/.github or the environment.", e); + // fall through + cause = e; } + + builder = fromEnvironment(); + + if (builder.user != null) + return builder; + else + throw new IOException("Failed to resolve credentials from ~/.github or the environment.", cause); } public static GitHubBuilder fromEnvironment(String loginVariableName, String passwordVariableName, String oauthVariableName) throws IOException { @@ -99,7 +90,7 @@ public static GitHubBuilder fromPropertyFile() throws IOException { File propertyFile = new File(homeDir, ".github"); return fromPropertyFile(propertyFile.getPath()); } - + public static GitHubBuilder fromPropertyFile(String propertyFileName) throws IOException { Properties props = new Properties(); FileInputStream in = null; From 0359160ac6ace1534fe02a7260a011910a50eb0c Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 15 Mar 2015 13:07:21 -0700 Subject: [PATCH 14/31] Avoided using JDK6 method --- src/main/java/org/kohsuke/github/GitHubBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/kohsuke/github/GitHubBuilder.java b/src/main/java/org/kohsuke/github/GitHubBuilder.java index f03875d122..2fa75d8303 100644 --- a/src/main/java/org/kohsuke/github/GitHubBuilder.java +++ b/src/main/java/org/kohsuke/github/GitHubBuilder.java @@ -57,7 +57,7 @@ public static GitHubBuilder fromCredentials() throws IOException { if (builder.user != null) return builder; else - throw new IOException("Failed to resolve credentials from ~/.github or the environment.", cause); + throw (IOException)new IOException("Failed to resolve credentials from ~/.github or the environment.").initCause(cause); } public static GitHubBuilder fromEnvironment(String loginVariableName, String passwordVariableName, String oauthVariableName) throws IOException { From 7ed234c8751e72451554242b8c5c8e9fb398e0e4 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 15 Mar 2015 13:17:57 -0700 Subject: [PATCH 15/31] Standardize environment variable names ... that are more like typical environment variables. Reasonably unique and upper case. Deprecate other methods. The point of a connector method is to make sure all clients of the same library uses the same environments, thereby eliminating the pain of setting credentials per app. Allowing the app to specify the environment variable names defeat this purpose. --- .../org/kohsuke/github/GitHubBuilder.java | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GitHubBuilder.java b/src/main/java/org/kohsuke/github/GitHubBuilder.java index 2fa75d8303..8390afb364 100644 --- a/src/main/java/org/kohsuke/github/GitHubBuilder.java +++ b/src/main/java/org/kohsuke/github/GitHubBuilder.java @@ -9,9 +9,13 @@ import java.net.HttpURLConnection; import java.net.Proxy; import java.net.URL; +import java.util.Locale; +import java.util.Map.Entry; import java.util.Properties; /** + * + * * @since 1.59 */ public class GitHubBuilder { @@ -60,6 +64,11 @@ public static GitHubBuilder fromCredentials() throws IOException { throw (IOException)new IOException("Failed to resolve credentials from ~/.github or the environment.").initCause(cause); } + /** + * @deprecated + * Use {@link #fromEnvironment()} to pick up standard set of environment variables, so that + * different clients of this library will all recognize one consistent set of coordinates. + */ public static GitHubBuilder fromEnvironment(String loginVariableName, String passwordVariableName, String oauthVariableName) throws IOException { return fromEnvironment(loginVariableName, passwordVariableName, oauthVariableName, ""); } @@ -70,6 +79,11 @@ private static void loadIfSet(String envName, Properties p, String propName) { p.put(propName, v); } + /** + * @deprecated + * Use {@link #fromEnvironment()} to pick up standard set of environment variables, so that + * different clients of this library will all recognize one consistent set of coordinates. + */ public static GitHubBuilder fromEnvironment(String loginVariableName, String passwordVariableName, String oauthVariableName, String endpointVariableName) throws IOException { Properties env = new Properties(); loadIfSet(loginVariableName,env,"login"); @@ -78,10 +92,34 @@ public static GitHubBuilder fromEnvironment(String loginVariableName, String pas loadIfSet(endpointVariableName,env,"endpoint"); return fromProperties(env); } - + + /** + * Creates {@link GitHubBuilder} by picking up coordinates from environment variables. + * + *

+ * The following environment variables are recognized: + * + *

    + *
  • GITHUB_LOGIN: username like 'kohsuke' + *
  • GITHUB_PASSWORD: raw password + *
  • GITHUB_OAUTH: OAuth token to login + *
  • GITHUB_ENDPOINT: URL of the API endpoint + *
+ * + *

+ * See class javadoc for the relationship between these coordinates. + * + *

+ * For backward compatibility, the following environment variables are recognized but discouraged: + * login, password, oauth + */ public static GitHubBuilder fromEnvironment() throws IOException { Properties props = new Properties(); - props.putAll(System.getenv()); + for (Entry e : System.getenv().entrySet()) { + String name = e.getKey().toLowerCase(Locale.ENGLISH); + if (name.startsWith("github_")) name=name.substring(7); + props.put(name,e.getValue()); + } return fromProperties(props); } From a4c1c8de24f7f1ab38b28e5e6f19cc46a5993d5d Mon Sep 17 00:00:00 2001 From: Kanstantsin Shautsou Date: Sun, 8 Mar 2015 02:58:36 +0300 Subject: [PATCH 16/31] Provide reset date info for rate limit --- .../java/org/kohsuke/github/GHRateLimit.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GHRateLimit.java b/src/main/java/org/kohsuke/github/GHRateLimit.java index 9d54953d54..301bd7993d 100644 --- a/src/main/java/org/kohsuke/github/GHRateLimit.java +++ b/src/main/java/org/kohsuke/github/GHRateLimit.java @@ -1,5 +1,7 @@ package org.kohsuke.github; +import java.util.Date; + /** * Rate limit. * @author Kohsuke Kawaguchi @@ -10,12 +12,28 @@ public class GHRateLimit { */ public int remaining; /** - * Alotted API call per hour. + * Allotted API call per hour. */ public int limit; + /** + * The time at which the current rate limit window resets in UTC epoch seconds. + */ + public Date reset; + + /** + * Non-epoch date + */ + public Date getResetDate() { + return new Date(reset.getTime() * 1000); + } + @Override public String toString() { - return remaining+"/"+limit; + return "GHRateLimit{" + + "remaining=" + remaining + + ", limit=" + limit + + ", resetDate=" + getResetDate() + + '}'; } } From 4093e53b5b7eb16e0b4de184a93262285ee6164b Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Tue, 17 Mar 2015 07:43:51 -0700 Subject: [PATCH 17/31] Implemented a strategy pattern to let the client determine API rate limit behavior. The default is set to the backward compatible behaviour. --- src/main/java/org/kohsuke/github/GitHub.java | 5 +- .../org/kohsuke/github/GitHubBuilder.java | 8 ++- .../org/kohsuke/github/RateLimitHandler.java | 61 +++++++++++++++++++ .../java/org/kohsuke/github/Requester.java | 11 +--- 4 files changed, 74 insertions(+), 11 deletions(-) create mode 100644 src/main/java/org/kohsuke/github/RateLimitHandler.java diff --git a/src/main/java/org/kohsuke/github/GitHub.java b/src/main/java/org/kohsuke/github/GitHub.java index c560227498..8217af5cde 100644 --- a/src/main/java/org/kohsuke/github/GitHub.java +++ b/src/main/java/org/kohsuke/github/GitHub.java @@ -75,6 +75,8 @@ public class GitHub { private final String apiUrl; + /*package*/ final RateLimitHandler rateLimitHandler; + private HttpConnector connector = HttpConnector.DEFAULT; /** @@ -113,7 +115,7 @@ public class GitHub { * @param connector * HttpConnector to use. Pass null to use default connector. */ - /* package */ GitHub(String apiUrl, String login, String oauthAccessToken, String password, HttpConnector connector) throws IOException { + /* package */ GitHub(String apiUrl, String login, String oauthAccessToken, String password, HttpConnector connector, RateLimitHandler rateLimitHandler) throws IOException { if (apiUrl.endsWith("/")) apiUrl = apiUrl.substring(0, apiUrl.length()-1); // normalize this.apiUrl = apiUrl; if (null != connector) this.connector = connector; @@ -132,6 +134,7 @@ public class GitHub { if (login==null && encodedAuthorization!=null) login = getMyself().getLogin(); this.login = login; + this.rateLimitHandler = rateLimitHandler; } /** diff --git a/src/main/java/org/kohsuke/github/GitHubBuilder.java b/src/main/java/org/kohsuke/github/GitHubBuilder.java index e794fb14ca..f35c765027 100644 --- a/src/main/java/org/kohsuke/github/GitHubBuilder.java +++ b/src/main/java/org/kohsuke/github/GitHubBuilder.java @@ -25,6 +25,8 @@ public class GitHubBuilder { private HttpConnector connector; + private RateLimitHandler rateLimitHandler = RateLimitHandler.WAIT; + public GitHubBuilder() { } @@ -155,6 +157,10 @@ public GitHubBuilder withConnector(HttpConnector connector) { this.connector = connector; return this; } + public GitHubBuilder withRateLimitHandler(RateLimitHandler handler) { + this.rateLimitHandler = handler; + return this; + } /** * Configures {@linkplain #withConnector(HttpConnector) connector} @@ -170,6 +176,6 @@ public HttpURLConnection connect(URL url) throws IOException { } public GitHub build() throws IOException { - return new GitHub(endpoint, user, oauthToken, password, connector); + return new GitHub(endpoint, user, oauthToken, password, connector, rateLimitHandler); } } diff --git a/src/main/java/org/kohsuke/github/RateLimitHandler.java b/src/main/java/org/kohsuke/github/RateLimitHandler.java new file mode 100644 index 0000000000..b00624ce2b --- /dev/null +++ b/src/main/java/org/kohsuke/github/RateLimitHandler.java @@ -0,0 +1,61 @@ +package org.kohsuke.github; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.HttpURLConnection; + +/** + * Pluggable strategy to determine what to do when the API rate limit is reached. + * + * @author Kohsuke Kawaguchi + * @see GitHubBuilder#withRateLimitHandler(RateLimitHandler) + */ +public abstract class RateLimitHandler { + /** + * Called when the library encounters HTTP error indicating that the API rate limit is reached. + * + *

+ * Any exception thrown from this method will cause the request to fail, and the caller of github-api + * will receive an exception. If this method returns normally, another request will be attempted. + * For that to make sense, the implementation needs to wait for some time. + * + * @see API documentation from GitHub + * @param e + * Exception from Java I/O layer. If you decide to fail the processing, you can throw + * this exception (or wrap this exception into another exception and throw it.) + * @param uc + * Connection that resulted in an error. Useful for accessing other response headers. + */ + public abstract void onError(IOException e, HttpURLConnection uc) throws IOException; + + /** + * Block until the API rate limit is reset. Useful for long-running batch processing. + */ + public static final RateLimitHandler WAIT = new RateLimitHandler() { + @Override + public void onError(IOException e, HttpURLConnection uc) throws IOException { + try { + Thread.sleep(parseWaitTime(uc)); + } catch (InterruptedException _) { + throw (InterruptedIOException)new InterruptedIOException().initCause(e); + } + } + + private long parseWaitTime(HttpURLConnection uc) { + String v = uc.getHeaderField("X-RateLimit-Reset"); + if (v==null) return 10000; // can't tell + + return Math.max(10000, Long.parseLong(v)*1000 - System.currentTimeMillis()); + } + }; + + /** + * Fail immediately. + */ + public static final RateLimitHandler FAIL = new RateLimitHandler() { + @Override + public void onError(IOException e, HttpURLConnection uc) throws IOException { + throw (IOException)new IOException("API rate limit reached").initCause(e); + } + }; +} diff --git a/src/main/java/org/kohsuke/github/Requester.java b/src/main/java/org/kohsuke/github/Requester.java index e2b0362a4c..896bbd605d 100644 --- a/src/main/java/org/kohsuke/github/Requester.java +++ b/src/main/java/org/kohsuke/github/Requester.java @@ -417,18 +417,11 @@ private InputStream wrapStream(HttpURLConnection uc, InputStream in) throws IOEx } /** - * If the error is because of the API limit, wait 10 sec and return normally. - * Otherwise throw an exception reporting an error. + * Handle API error by either throwing it or by returning normally to retry. */ /*package*/ void handleApiError(IOException e, HttpURLConnection uc) throws IOException { if ("0".equals(uc.getHeaderField("X-RateLimit-Remaining"))) { - // API limit reached. wait 10 secs and return normally - try { - Thread.sleep(10000); - return; - } catch (InterruptedException _) { - throw (InterruptedIOException)new InterruptedIOException().initCause(e); - } + root.rateLimitHandler.onError(e,uc); } if (e instanceof FileNotFoundException) From 39b32cee2e791386c6c43b2404bee472fa338e2f Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 21 Mar 2015 16:35:34 -0700 Subject: [PATCH 18/31] Implemented /repositories Fixed issue #157 --- src/main/java/org/kohsuke/github/GitHub.java | 28 +++++++++++++++++++ src/test/java/org/kohsuke/github/AppTest.java | 12 ++++++++ 2 files changed, 40 insertions(+) diff --git a/src/main/java/org/kohsuke/github/GitHub.java b/src/main/java/org/kohsuke/github/GitHub.java index 8217af5cde..a5071457b7 100644 --- a/src/main/java/org/kohsuke/github/GitHub.java +++ b/src/main/java/org/kohsuke/github/GitHub.java @@ -441,6 +441,34 @@ public GHIssueSearchBuilder searchIssues() { return new GHIssueSearchBuilder(this); } + /** + * This provides a dump of every public repository, in the order that they were created. + * @see documentation + */ + public PagedIterable listAllPublicRepositories() { + return listAllPublicRepositories(null); + } + + /** + * This provides a dump of every public repository, in the order that they were created. + * + * @param since + * The integer ID of the last Repository that you’ve seen. See {@link GHRepository#getId()} + * @see documentation + */ + public PagedIterable listAllPublicRepositories(final String since) { + return new PagedIterable() { + public PagedIterator iterator() { + return new PagedIterator(retrieve().with("since",since).asIterator("/repositories", GHRepository[].class)) { + @Override + protected void wrapUp(GHRepository[] page) { + for (GHRepository c : page) + c.wrap(GitHub.this); + } + }; + } + }; + } /*package*/ static URL parseURL(String s) { try { diff --git a/src/test/java/org/kohsuke/github/AppTest.java b/src/test/java/org/kohsuke/github/AppTest.java index 36d1825882..fed7aec954 100755 --- a/src/test/java/org/kohsuke/github/AppTest.java +++ b/src/test/java/org/kohsuke/github/AppTest.java @@ -737,6 +737,18 @@ public void testSubscribers() throws IOException { assertTrue(githubApi); } + @Test + public void testListAllRepositories() throws Exception { + Iterator itr = gitHub.listAllPublicRepositories().iterator(); + for (int i=0; i<30; i++) { + assertTrue(itr.hasNext()); + GHRepository r = itr.next(); + System.out.println(r.getFullName()); + assertNotNull(r.getUrl()); + assertNotEquals(0,r.getId()); + } + } + private void kohsuke() { String login = getUser().getLogin(); Assume.assumeTrue(login.equals("kohsuke") || login.equals("kohsuke2")); From 72fc31313572b3d8564034729ad31cc367cf5329 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 21 Mar 2015 16:43:11 -0700 Subject: [PATCH 19/31] improved error handling --- src/main/java/org/kohsuke/github/Requester.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/kohsuke/github/Requester.java b/src/main/java/org/kohsuke/github/Requester.java index 896bbd605d..647a258139 100644 --- a/src/main/java/org/kohsuke/github/Requester.java +++ b/src/main/java/org/kohsuke/github/Requester.java @@ -52,6 +52,7 @@ import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; +import com.fasterxml.jackson.databind.JsonMappingException; import org.apache.commons.io.IOUtils; import javax.net.ssl.HttpsURLConnection; @@ -396,7 +397,11 @@ private T parse(HttpURLConnection uc, Class type, T instance) throws IOEx r = new InputStreamReader(wrapStream(uc, uc.getInputStream()), "UTF-8"); String data = IOUtils.toString(r); if (type!=null) - return MAPPER.readValue(data,type); + try { + return MAPPER.readValue(data,type); + } catch (JsonMappingException e) { + throw (IOException)new IOException("Failed to deserialize "+data).initCause(e); + } if (instance!=null) return MAPPER.readerForUpdating(instance).readValue(data); return null; From 77590b4eb3c468ee4ead0f8cc21bcab120819984 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 21 Mar 2015 16:52:49 -0700 Subject: [PATCH 20/31] eliminate the need for path manipulation and consolidate them to 'with' --- src/main/java/org/kohsuke/github/GHRepository.java | 10 ++-------- src/main/java/org/kohsuke/github/Requester.java | 8 ++++++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GHRepository.java b/src/main/java/org/kohsuke/github/GHRepository.java index 85d4592ce3..7c7e55d328 100644 --- a/src/main/java/org/kohsuke/github/GHRepository.java +++ b/src/main/java/org/kohsuke/github/GHRepository.java @@ -1029,10 +1029,7 @@ public GHContent getFileContent(String path, String ref) throws IOException { Requester requester = root.retrieve(); String target = getApiTailUrl("contents/" + path); - if (ref != null) - target = target + "?ref=" + ref; - - return requester.to(target, GHContent.class).wrap(this); + return requester.with("ref",ref).to(target, GHContent.class).wrap(this); } public List getDirectoryContent(String path) throws IOException { @@ -1043,10 +1040,7 @@ public List getDirectoryContent(String path, String ref) throws IOExc Requester requester = root.retrieve(); String target = getApiTailUrl("contents/" + path); - if (ref != null) - target = target + "?ref=" + ref; - - GHContent[] files = requester.to(target, GHContent[].class); + GHContent[] files = requester.with("ref",ref).to(target, GHContent[].class); GHContent.wrap(files, this); diff --git a/src/main/java/org/kohsuke/github/Requester.java b/src/main/java/org/kohsuke/github/Requester.java index 647a258139..4dcc631421 100644 --- a/src/main/java/org/kohsuke/github/Requester.java +++ b/src/main/java/org/kohsuke/github/Requester.java @@ -191,6 +191,14 @@ public T to(String tailApiUrl, Class type, String method) throws IOExcept private T _to(String tailApiUrl, Class type, T instance) throws IOException { while (true) {// loop while API rate limit is hit + if (method.equals("GET") && !args.isEmpty()) { + StringBuilder qs=new StringBuilder(); + for (Entry arg : args) { + qs.append(qs.length()==0 ? '?' : '&'); + qs.append(arg.key).append('=').append(URLEncoder.encode(arg.value.toString(),"UTF-8")); + } + tailApiUrl += qs.toString(); + } HttpURLConnection uc = setupConnection(root.getApiURL(tailApiUrl)); buildRequest(uc); From 709e47f32fa763895bcee113cd189e01dc82816d Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 22 Mar 2015 09:10:34 -0700 Subject: [PATCH 21/31] Added getDownloadUrl() method --- src/main/java/org/kohsuke/github/GHContent.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GHContent.java b/src/main/java/org/kohsuke/github/GHContent.java index 4900a19214..fb12dd0070 100644 --- a/src/main/java/org/kohsuke/github/GHContent.java +++ b/src/main/java/org/kohsuke/github/GHContent.java @@ -1,8 +1,6 @@ package org.kohsuke.github; import java.io.IOException; -import java.util.Arrays; -import java.util.List; import javax.xml.bind.DatatypeConverter; @@ -26,6 +24,7 @@ public class GHContent { private String url; // this is the API url private String git_url; // this is the Blob url private String html_url; // this is the UI + private String download_url; public GHRepository getOwner() { return owner; @@ -101,6 +100,11 @@ public String getHtmlUrl() { return html_url; } + /** + * URL to retrieve the raw content of the file. Null if this is a directory. + */ + public String getDownloadUrl() { return download_url; } + public boolean isFile() { return "file".equals(type); } From 5a8845f7f64bd5869d40d21cdd918f064a7bac0f Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 22 Mar 2015 09:17:49 -0700 Subject: [PATCH 22/31] added a method to return the raw unprocessed body --- src/main/java/org/kohsuke/github/Requester.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/kohsuke/github/Requester.java b/src/main/java/org/kohsuke/github/Requester.java index 4dcc631421..595b844dd1 100644 --- a/src/main/java/org/kohsuke/github/Requester.java +++ b/src/main/java/org/kohsuke/github/Requester.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.InterruptedIOException; import java.io.Reader; import java.io.UnsupportedEncodingException; import java.lang.reflect.Array; @@ -55,8 +54,6 @@ import com.fasterxml.jackson.databind.JsonMappingException; import org.apache.commons.io.IOUtils; -import javax.net.ssl.HttpsURLConnection; - /** * A builder pattern for making HTTP call and parsing its output. * @@ -247,6 +244,20 @@ public int asHttpStatusCode(String tailApiUrl) throws IOException { } } + public InputStream read(String tailApiUrl) throws IOException { + while (true) {// loop while API rate limit is hit + HttpURLConnection uc = setupConnection(root.getApiURL(tailApiUrl)); + + buildRequest(uc); + + try { + return uc.getInputStream(); + } catch (IOException e) { + handleApiError(e,uc); + } + } + } + private void buildRequest(HttpURLConnection uc) throws IOException { if (!method.equals("GET")) { uc.setDoOutput(true); From 86b0d272994f471ac983de400593134d269c3f6d Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 22 Mar 2015 10:21:09 -0700 Subject: [PATCH 23/31] added a method to read content. This also fixes #162. --- .../java/org/kohsuke/github/GHContent.java | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GHContent.java b/src/main/java/org/kohsuke/github/GHContent.java index fb12dd0070..cd04e436f1 100644 --- a/src/main/java/org/kohsuke/github/GHContent.java +++ b/src/main/java/org/kohsuke/github/GHContent.java @@ -1,6 +1,10 @@ package org.kohsuke.github; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.IOUtils; + import java.io.IOException; +import java.io.InputStream; import javax.xml.bind.DatatypeConverter; @@ -57,35 +61,34 @@ public String getPath() { /** * Retrieve the decoded content that is stored at this location. * + *

* Due to the nature of GitHub's API, you're not guaranteed that * the content will already be populated, so this may trigger * network activity, and can throw an IOException. - **/ + * + * @deprecated + * Use {@link #read()} + */ public String getContent() throws IOException { return new String(DatatypeConverter.parseBase64Binary(getEncodedContent())); } /** - * Retrieve the raw content that is stored at this location. + * Retrieve the base64-encoded content that is stored at this location. * + *

* Due to the nature of GitHub's API, you're not guaranteed that * the content will already be populated, so this may trigger * network activity, and can throw an IOException. - **/ + * + * @deprecated + * Use {@link #read()} + */ public String getEncodedContent() throws IOException { - if (content != null) + if (content!=null) return content; - - GHContent retrievedContent = owner.getFileContent(path); - - this.size = retrievedContent.size; - this.sha = retrievedContent.sha; - this.content = retrievedContent.content; - this.url = retrievedContent.url; - this.git_url = retrievedContent.git_url; - this.html_url = retrievedContent.html_url; - - return content; + else + return Base64.encodeBase64String(IOUtils.toByteArray(read())); } public String getUrl() { @@ -100,6 +103,13 @@ public String getHtmlUrl() { return html_url; } + /** + * Retrieves the actual content stored here. + */ + public InputStream read() throws IOException { + return new Requester(owner.root).read(getDownloadUrl()); + } + /** * URL to retrieve the raw content of the file. Null if this is a directory. */ From 6f4832476a4d1f76659f68ebc3c3a7e3d0b99107 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 22 Mar 2015 10:21:16 -0700 Subject: [PATCH 24/31] test case for #162 --- src/test/java/org/kohsuke/github/AppTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/test/java/org/kohsuke/github/AppTest.java b/src/test/java/org/kohsuke/github/AppTest.java index fed7aec954..806d473223 100755 --- a/src/test/java/org/kohsuke/github/AppTest.java +++ b/src/test/java/org/kohsuke/github/AppTest.java @@ -749,6 +749,20 @@ public void testListAllRepositories() throws Exception { } } + @Test // issue #162 + public void testIssue162() throws Exception { + GHRepository r = gitHub.getRepository("kohsuke/github-api"); + List contents = r.getDirectoryContent("", "gh-pages"); + for (GHContent content : contents) { + if (content.isFile()) { + String content1 = content.getContent(); + String content2 = r.getFileContent(content.getPath(), "gh-pages").getContent(); + System.out.println(content.getPath()); + assertEquals(content1,content2); + } + } + } + private void kohsuke() { String login = getUser().getLogin(); Assume.assumeTrue(login.equals("kohsuke") || login.equals("kohsuke2")); From 75512ff66ab13aa759750e51214bcf9c5f59b67c Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 22 Mar 2015 10:38:57 -0700 Subject: [PATCH 25/31] Turns out the interning of GHUser wasn't working at all! Fixes issue #166. --- src/main/java/org/kohsuke/github/GitHub.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/kohsuke/github/GitHub.java b/src/main/java/org/kohsuke/github/GitHub.java index a5071457b7..e94a47c949 100644 --- a/src/main/java/org/kohsuke/github/GitHub.java +++ b/src/main/java/org/kohsuke/github/GitHub.java @@ -293,7 +293,7 @@ protected GHUser getUser(GHUser orig) throws IOException { GHUser u = users.get(orig.getLogin()); if (u==null) { orig.root = this; - users.put(login,orig); + users.put(orig.getLogin(),orig); return orig; } return u; From 5bf252e12d7aa20f215bfbd8828835be91ffd777 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 22 Mar 2015 10:59:16 -0700 Subject: [PATCH 26/31] Added markdown support Fixes issue #165 --- .../java/org/kohsuke/github/GHRepository.java | 22 +++++++++++++- src/main/java/org/kohsuke/github/GitHub.java | 21 ++++++++++++++ .../java/org/kohsuke/github/MarkdownMode.java | 29 +++++++++++++++++++ .../java/org/kohsuke/github/Requester.java | 2 +- src/test/java/org/kohsuke/github/AppTest.java | 16 +++++++++- 5 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/kohsuke/github/MarkdownMode.java diff --git a/src/main/java/org/kohsuke/github/GHRepository.java b/src/main/java/org/kohsuke/github/GHRepository.java index 7c7e55d328..1c9fcc7b34 100644 --- a/src/main/java/org/kohsuke/github/GHRepository.java +++ b/src/main/java/org/kohsuke/github/GHRepository.java @@ -30,7 +30,9 @@ import javax.xml.bind.DatatypeConverter; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStreamReader; import java.io.InterruptedIOException; +import java.io.Reader; import java.net.URL; import java.util.*; @@ -1149,7 +1151,25 @@ public int getContributions() { return contributions; } } - + + /** + * Render a Markdown document. + * + * In {@linkplain MarkdownMode#GFM GFM mode}, issue numbers and user mentions + * are linked accordingly. + * + * @see GitHub#renderMarkdown(String) + */ + public Reader renderMarkdown(String text, MarkdownMode mode) throws IOException { + return new InputStreamReader( + new Requester(root) + .with("text", text) + .with("mode",mode==null?null:mode.toString()) + .with("context", getFullName()) + .read("/markdown"), + "UTF-8"); + } + @Override diff --git a/src/main/java/org/kohsuke/github/GitHub.java b/src/main/java/org/kohsuke/github/GitHub.java index e94a47c949..4781f69153 100644 --- a/src/main/java/org/kohsuke/github/GitHub.java +++ b/src/main/java/org/kohsuke/github/GitHub.java @@ -26,8 +26,10 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.ANY; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStreamReader; import java.io.Reader; import java.net.MalformedURLException; import java.net.URL; @@ -470,6 +472,25 @@ protected void wrapUp(GHRepository[] page) { }; } + /** + * Render a Markdown document in raw mode. + * + *

+ * It takes a Markdown document as plaintext and renders it as plain Markdown + * without a repository context (just like a README.md file is rendered – this + * is the simplest way to preview a readme online). + * + * @see GHRepository#renderMarkdown(String, MarkdownMode) + */ + public Reader renderMarkdown(String text) throws IOException { + return new InputStreamReader( + new Requester(this) + .with(new ByteArrayInputStream(text.getBytes("UTF-8"))) + .contentType("text/plain;charset=UTF-8") + .read("/markdown/raw"), + "UTF-8"); + } + /*package*/ static URL parseURL(String s) { try { return s==null ? null : new URL(s); diff --git a/src/main/java/org/kohsuke/github/MarkdownMode.java b/src/main/java/org/kohsuke/github/MarkdownMode.java new file mode 100644 index 0000000000..1d63760555 --- /dev/null +++ b/src/main/java/org/kohsuke/github/MarkdownMode.java @@ -0,0 +1,29 @@ +package org.kohsuke.github; + +import java.util.Locale; + +/** + * Rendering mode of markdown. + * + * @author Kohsuke Kawaguchi + * @see GitHub#renderMarkdown(String) + * @see GHRepository#renderMarkdown(String, MarkdownMode) + */ +public enum MarkdownMode { + /** + * Render a document as plain Markdown, just like README files are rendered. + */ + MARKDOWN, + /** + * Render a document as user-content, e.g. like user comments or issues are rendered. + * In GFM mode, hard line breaks are always taken into account, and issue and user + * mentions are linked accordingly. + * + * @see GHRepository#renderMarkdown(String, MarkdownMode) + */ + GFM; + + public String toString() { + return name().toLowerCase(Locale.ENGLISH); + } +} diff --git a/src/main/java/org/kohsuke/github/Requester.java b/src/main/java/org/kohsuke/github/Requester.java index 595b844dd1..7f34e72144 100644 --- a/src/main/java/org/kohsuke/github/Requester.java +++ b/src/main/java/org/kohsuke/github/Requester.java @@ -251,7 +251,7 @@ public InputStream read(String tailApiUrl) throws IOException { buildRequest(uc); try { - return uc.getInputStream(); + return wrapStream(uc,uc.getInputStream()); } catch (IOException e) { handleApiError(e,uc); } diff --git a/src/test/java/org/kohsuke/github/AppTest.java b/src/test/java/org/kohsuke/github/AppTest.java index 806d473223..31dfb52607 100755 --- a/src/test/java/org/kohsuke/github/AppTest.java +++ b/src/test/java/org/kohsuke/github/AppTest.java @@ -4,6 +4,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import org.apache.commons.io.IOUtils; import org.junit.Assume; import org.junit.Test; import org.kohsuke.github.GHCommit.File; @@ -758,11 +759,24 @@ public void testIssue162() throws Exception { String content1 = content.getContent(); String content2 = r.getFileContent(content.getPath(), "gh-pages").getContent(); System.out.println(content.getPath()); - assertEquals(content1,content2); + assertEquals(content1, content2); } } } + @Test + public void markDown() throws Exception { + assertEquals("

Test日本語

", IOUtils.toString(gitHub.renderMarkdown("**Test日本語**")).trim()); + + String actual = IOUtils.toString(gitHub.getRepository("kohsuke/github-api").renderMarkdown("@kohsuke to fix issue #1", MarkdownMode.GFM)); + System.out.println(actual); + assertTrue(actual.contains("href=\"https://github.com/kohsuke\"")); + assertTrue(actual.contains("href=\"https://github.com/kohsuke/github-api/pull/1\"")); + assertTrue(actual.contains("class=\"user-mention\"")); + assertTrue(actual.contains("class=\"issue-link\"")); + assertTrue(actual.contains("to fix issue")); + } + private void kohsuke() { String login = getUser().getLogin(); Assume.assumeTrue(login.equals("kohsuke") || login.equals("kohsuke2")); From cc84c867c029e0e936138a38fc9b3d98dd57dc26 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 22 Mar 2015 11:00:38 -0700 Subject: [PATCH 27/31] I think our coverage is pretty good now --- src/site/markdown/index.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/site/markdown/index.md b/src/site/markdown/index.md index 3956df708a..3c2af01970 100644 --- a/src/site/markdown/index.md +++ b/src/site/markdown/index.md @@ -9,8 +9,7 @@ are used in favor of using string handle (such as `GHUser.isMemberOf(GHOrganizat The library supports both github.com and GitHub Enterprise. -There are some corners of the GitHub API that's not yet implemented, but -the library is implemented with the right abstractions and libraries to make it very easy to improve the coverage. +Most of the GitHub APIs are covered, although there are some corners that are still not yet implemented. Sample Usage ----- From 5c7b259fe92fa275d4f81673ca08755e45861808 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 22 Mar 2015 11:02:17 -0700 Subject: [PATCH 28/31] Using the latest --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ebfe7dc7b0..ec7557ffb0 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.kohsuke pom - 12 + 14 github-api From 6229e0928dc121cec9d43d84759a9a55a9ed5f61 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 22 Mar 2015 11:11:03 -0700 Subject: [PATCH 29/31] Revert "Set credentials file according to documentation" This reverts commit 0bf81f4fb9d34633bea1e88877d67a9a2caf554e. The point of this is to allow me to use a separate account to avoid corrupting my event stream. GitHub.connect() does the standard handling for those who are not me. --- src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java b/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java index 0f3145e4fa..e45895d66e 100644 --- a/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java +++ b/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java @@ -18,7 +18,7 @@ public abstract class AbstractGitHubApiTestBase extends Assert { @Before public void setUp() throws Exception { Properties props = new Properties(); - java.io.File f = new java.io.File(System.getProperty("user.home"), ".github"); + java.io.File f = new java.io.File(System.getProperty("user.home"), ".github.kohsuke2"); if (f.exists()) { FileInputStream in = new FileInputStream(f); try { From 10238dbcd34ada9d6a2175a996502ca75abe9a5c Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 22 Mar 2015 11:13:40 -0700 Subject: [PATCH 30/31] Explaining why this code is the way it is. --- .../github/AbstractGitHubApiTestBase.java | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java b/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java index e45895d66e..a4311429d7 100644 --- a/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java +++ b/src/test/java/org/kohsuke/github/AbstractGitHubApiTestBase.java @@ -1,12 +1,10 @@ package org.kohsuke.github; -import org.apache.commons.io.IOUtils; import org.junit.Assert; import org.junit.Before; import org.kohsuke.randname.RandomNameGenerator; -import java.io.FileInputStream; -import java.util.Properties; +import java.io.File; /** * @author Kohsuke Kawaguchi @@ -17,16 +15,11 @@ public abstract class AbstractGitHubApiTestBase extends Assert { @Before public void setUp() throws Exception { - Properties props = new Properties(); - java.io.File f = new java.io.File(System.getProperty("user.home"), ".github.kohsuke2"); + File f = new File(System.getProperty("user.home"), ".github.kohsuke2"); if (f.exists()) { - FileInputStream in = new FileInputStream(f); - try { - props.load(in); - gitHub = GitHub.connect(props.getProperty("login"),props.getProperty("oauth")); - } finally { - IOUtils.closeQuietly(in); - } + // use the non-standard credential preferentially, so that developers of this library do not have + // to clutter their event stream. + gitHub = GitHubBuilder.fromPropertyFile(f.getPath()).build(); } else { gitHub = GitHub.connect(); } From 8b428f2c93fd1b127dad925273071625d3894e91 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 22 Mar 2015 11:14:19 -0700 Subject: [PATCH 31/31] whitespace only changes for consistent indentation --- src/main/java/org/kohsuke/github/GitHub.java | 42 +++++++++---------- src/test/java/org/kohsuke/github/AppTest.java | 34 +++++++-------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/kohsuke/github/GitHub.java b/src/main/java/org/kohsuke/github/GitHub.java index 4781f69153..ec1a2a46d5 100644 --- a/src/main/java/org/kohsuke/github/GitHub.java +++ b/src/main/java/org/kohsuke/github/GitHub.java @@ -336,27 +336,27 @@ public Map getMyOrganizations() throws IOException { return r; } - /** - * Gets complete map of organizations/teams that current user belongs to. - * - * Leverages the new GitHub API /user/teams made available recently to - * get in a single call the complete set of organizations, teams and permissions - * in a single call. - */ - public Map> getMyTeams() throws IOException { - Map> allMyTeams = new HashMap>(); - for (GHTeam team : retrieve().to("/user/teams", GHTeam[].class)) { - team.wrapUp(this); - String orgLogin = team.getOrganization().getLogin(); - Set teamsPerOrg = allMyTeams.get(orgLogin); - if (teamsPerOrg == null) { - teamsPerOrg = new HashSet(); - } - teamsPerOrg.add(team); - allMyTeams.put(orgLogin, teamsPerOrg); - } - return allMyTeams; - } + /** + * Gets complete map of organizations/teams that current user belongs to. + * + * Leverages the new GitHub API /user/teams made available recently to + * get in a single call the complete set of organizations, teams and permissions + * in a single call. + */ + public Map> getMyTeams() throws IOException { + Map> allMyTeams = new HashMap>(); + for (GHTeam team : retrieve().to("/user/teams", GHTeam[].class)) { + team.wrapUp(this); + String orgLogin = team.getOrganization().getLogin(); + Set teamsPerOrg = allMyTeams.get(orgLogin); + if (teamsPerOrg == null) { + teamsPerOrg = new HashSet(); + } + teamsPerOrg.add(team); + allMyTeams.put(orgLogin, teamsPerOrg); + } + return allMyTeams; + } /** * Public events visible to you. Equivalent of what's displayed on https://github.com/ diff --git a/src/test/java/org/kohsuke/github/AppTest.java b/src/test/java/org/kohsuke/github/AppTest.java index 31dfb52607..6e56bbff89 100755 --- a/src/test/java/org/kohsuke/github/AppTest.java +++ b/src/test/java/org/kohsuke/github/AppTest.java @@ -207,31 +207,31 @@ public void testMyOrganizations() throws IOException { @Test public void testMyTeamsContainsAllMyOrganizations() throws IOException { - Map> teams = gitHub.getMyTeams(); - Map myOrganizations = gitHub.getMyOrganizations(); - assertEquals(teams.keySet(), myOrganizations.keySet()); + Map> teams = gitHub.getMyTeams(); + Map myOrganizations = gitHub.getMyOrganizations(); + assertEquals(teams.keySet(), myOrganizations.keySet()); } @Test public void testMyTeamsShouldIncludeMyself() throws IOException { - Map> teams = gitHub.getMyTeams(); - for (Entry> teamsPerOrg : teams.entrySet()) { - String organizationName = teamsPerOrg.getKey(); - for (GHTeam team : teamsPerOrg.getValue()) { - String teamName = team.getName(); - assertTrue("Team " + teamName + " in organization " + organizationName - + " does not contain myself", - shouldBelongToTeam(organizationName, teamName)); + Map> teams = gitHub.getMyTeams(); + for (Entry> teamsPerOrg : teams.entrySet()) { + String organizationName = teamsPerOrg.getKey(); + for (GHTeam team : teamsPerOrg.getValue()) { + String teamName = team.getName(); + assertTrue("Team " + teamName + " in organization " + organizationName + + " does not contain myself", + shouldBelongToTeam(organizationName, teamName)); + } } - } } private boolean shouldBelongToTeam(String organizationName, String teamName) throws IOException { - GHOrganization org = gitHub.getOrganization(organizationName); - assertNotNull(org); - GHTeam team = org.getTeamByName(teamName); - assertNotNull(team); - return team.hasMember(gitHub.getMyself()); + GHOrganization org = gitHub.getOrganization(organizationName); + assertNotNull(org); + GHTeam team = org.getTeamByName(teamName); + assertNotNull(team); + return team.hasMember(gitHub.getMyself()); } @Test