batch : iterateRepositories(100)) {
+ for (GHRepository r : batch)
repositories.put(r.getName(),r);
- }
- if (array.length==0) break;
}
-
return Collections.unmodifiableMap(repositories);
}
+ /**
+ * Loads repository list in a pagenated fashion.
+ *
+ *
+ * For a person with a lot of repositories, GitHub returns the list of repositories in a pagenated fashion.
+ * Unlike {@link #getRepositories()}, this method allows the caller to start processing data as it arrives.
+ *
+ * Every {@link Iterator#next()} call results in I/O. Exceptions that occur during the processing is wrapped
+ * into {@link Error}.
+ */
+ public synchronized Iterable> iterateRepositories(final int pageSize) {
+ return new Iterable>() {
+ public Iterator> iterator() {
+ final Iterator pager = root.retrievePaged("/users/" + login + "/repos?per_page="+pageSize,GHRepository[].class,false, V3);
+
+ return new Iterator>() {
+ public boolean hasNext() {
+ return pager.hasNext();
+ }
+
+ public List next() {
+ GHRepository[] batch = pager.next();
+ for (GHRepository r : batch)
+ r.root = root;
+ return Arrays.asList(batch);
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ };
+ }
+
/**
*
* @return
diff --git a/src/main/java/org/kohsuke/github/GitHub.java b/src/main/java/org/kohsuke/github/GitHub.java
index 76b2684f15..890ee993cf 100644
--- a/src/main/java/org/kohsuke/github/GitHub.java
+++ b/src/main/java/org/kohsuke/github/GitHub.java
@@ -44,13 +44,17 @@
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.TimeZone;
+import java.util.zip.GZIPInputStream;
import com.infradna.tool.bridge_method_injector.WithBridgeMethods;
import org.apache.commons.io.IOUtils;
+import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.map.DeserializationConfig.Feature;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.introspect.VisibilityChecker.Std;
@@ -170,7 +174,7 @@ public static GitHub connectAnonymously() {
}
/*package*/ T retrieveWithAuth(String tailApiUrl, Class type) throws IOException {
- return retrieveWithAuth(tailApiUrl,type,"GET");
+ return retrieveWithAuth(tailApiUrl, type, "GET");
}
/*package*/ T retrieveWithAuth3(String tailApiUrl, Class type) throws IOException {
@@ -187,32 +191,141 @@ public static GitHub connectAnonymously() {
private T _retrieve(String tailApiUrl, Class type, String method, boolean withAuth, ApiVersion v) throws IOException {
while (true) {// loop while API rate limit is hit
-
- HttpURLConnection uc = (HttpURLConnection) getApiURL(v,tailApiUrl).openConnection();
-
- if (withAuth && this.oauthAccessToken == null)
- uc.setRequestProperty("Authorization", "Basic " + encodedAuthorization);
-
- uc.setRequestMethod(method);
- if (method.equals("PUT")) {
- uc.setDoOutput(true);
- uc.setRequestProperty("Content-Length","0");
- uc.getOutputStream().close();
- }
-
+ HttpURLConnection uc = setupConnection(method, withAuth, getApiURL(v, tailApiUrl));
try {
- InputStreamReader r = new InputStreamReader(uc.getInputStream(), "UTF-8");
- if (type==null) {
- String data = IOUtils.toString(r);
- return null;
- }
- return MAPPER.readValue(r,type);
+ return parse(uc,type);
} catch (IOException e) {
handleApiError(e,uc);
}
}
}
+ /**
+ * Loads pagenated resources.
+ *
+ * Every iterator call reports a new batch.
+ */
+ /*package*/ Iterator retrievePaged(final String tailApiUrl, final Class type, final boolean withAuth, final ApiVersion v) {
+ return new Iterator() {
+ /**
+ * The next batch to be returned from {@link #next()}.
+ */
+ T next;
+ /**
+ * URL of the next resource to be retrieved, or null if no more data is available.
+ */
+ URL url;
+
+ {
+ try {
+ url = getApiURL(v, tailApiUrl);
+ } catch (IOException e) {
+ throw new Error(e);
+ }
+ }
+
+ public boolean hasNext() {
+ fetch();
+ return next!=null;
+ }
+
+ public T next() {
+ fetch();
+ T r = next;
+ if (r==null) throw new NoSuchElementException();
+ next = null;
+ return r;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ private void fetch() {
+ if (next!=null) return; // already fetched
+ if (url==null) return; // no more data to fetch
+
+ try {
+ while (true) {// loop while API rate limit is hit
+ HttpURLConnection uc = setupConnection("GET", withAuth, url);
+ try {
+ next = parse(uc,type);
+ assert next!=null;
+ findNextURL(uc);
+ return;
+ } catch (IOException e) {
+ handleApiError(e,uc);
+ }
+ }
+ } catch (IOException e) {
+ throw new Error(e);
+ }
+ }
+
+ /**
+ * Locate the next page from the pagination "Link" tag.
+ */
+ private void findNextURL(HttpURLConnection uc) throws MalformedURLException {
+ url = null; // start defensively
+ String link = uc.getHeaderField("Link");
+ if (link==null) return;
+
+ for (String token : link.split(", ")) {
+ if (token.endsWith("rel=\"next\"")) {
+ // found the next page. This should look something like
+ // ; rel="next"
+ int idx = token.indexOf('>');
+ url = new URL(token.substring(1,idx));
+ return;
+ }
+ }
+
+ // no more "next" link. we are done.
+ }
+ };
+ }
+
+ private HttpURLConnection setupConnection(String method, boolean withAuth, URL url) throws IOException {
+ HttpURLConnection uc = (HttpURLConnection) url.openConnection();
+
+ if (withAuth && this.oauthAccessToken == null)
+ uc.setRequestProperty("Authorization", "Basic " + encodedAuthorization);
+
+ uc.setRequestMethod(method);
+ if (method.equals("PUT")) {
+ uc.setDoOutput(true);
+ uc.setRequestProperty("Content-Length","0");
+ uc.getOutputStream().close();
+ }
+ uc.setRequestProperty("Accept-Encoding", "gzip");
+ return uc;
+ }
+
+ private T parse(HttpURLConnection uc, Class type) throws IOException {
+ InputStreamReader r = null;
+ try {
+ r = new InputStreamReader(wrapStream(uc, uc.getInputStream()), "UTF-8");
+ if (type==null) {
+ String data = IOUtils.toString(r);
+ return null;
+ }
+ return MAPPER.readValue(r,type);
+ } finally {
+ IOUtils.closeQuietly(r);
+ }
+ }
+
+ /**
+ * Handles the "Content-Encoding" header.
+ */
+ private InputStream wrapStream(HttpURLConnection uc, InputStream in) throws IOException {
+ String encoding = uc.getContentEncoding();
+ if (encoding==null) return in;
+ if (encoding.equals("gzip")) return new GZIPInputStream(in);
+
+ throw new UnsupportedOperationException("Unexpected Content-Encoding: "+encoding);
+ }
+
/**
* If the error is because of the API limit, wait 10 sec and return normally.
* Otherwise throw an exception reporting an error.
@@ -231,11 +344,15 @@ private T _retrieve(String tailApiUrl, Class type, String method, boolean
if (e instanceof FileNotFoundException)
throw e; // pass through 404 Not Found to allow the caller to handle it intelligently
- InputStream es = uc.getErrorStream();
- if (es!=null)
- throw (IOException)new IOException(IOUtils.toString(es,"UTF-8")).initCause(e);
- else
- throw e;
+ InputStream es = wrapStream(uc, uc.getErrorStream());
+ try {
+ if (es!=null)
+ throw (IOException)new IOException(IOUtils.toString(es,"UTF-8")).initCause(e);
+ else
+ throw e;
+ } finally {
+ IOUtils.closeQuietly(es);
+ }
}
/**
diff --git a/src/test/java/org/kohsuke/AppTest.java b/src/test/java/org/kohsuke/AppTest.java
index 8260a1dce0..41e72589f1 100644
--- a/src/test/java/org/kohsuke/AppTest.java
+++ b/src/test/java/org/kohsuke/AppTest.java
@@ -192,6 +192,15 @@ private void testPostCommitHook(GitHub gitHub) throws IOException {
System.out.println(hooks);
}
+ public void testOrgRepositories() throws IOException {
+ GitHub gitHub = GitHub.connect();
+ GHOrganization j = gitHub.getOrganization("jenkinsci");
+ long start = System.currentTimeMillis();
+ Map repos = j.getRepositories();
+ long end = System.currentTimeMillis();
+ System.out.printf("%d repositories in %dms\n",repos.size(),end-start);
+ }
+
public void testOrganization() throws IOException {
GitHub gitHub = GitHub.connect();
GHOrganization j = gitHub.getOrganization("jenkinsci");