From aa4d41b70b2a66c3de5a7dd19b0f7c151f906630 Mon Sep 17 00:00:00 2001 From: Andrew Tergis Date: Tue, 3 Dec 2019 16:14:24 -0500 Subject: [PATCH 1/2] feat: add support for /import/github Addresses python-gitlab/python-gitlab#952 This adds a method to the `ProjectManager` called `import_github`, which maps to the `/import/github` API endpoint. Calling `import_github` will trigger an import operation from into , using to authenticate against github. In practice a gitlab server may take many 10's of seconds to respond to this API call, so we also take the liberty of increasing the default timeout (only for this method invocation). Unfortunately since `import` is a protected keyword in python, I was unable to follow the endpoint structure with the manager namespace. I'm open to suggestions on a more sensible interface. I'm successfully using this addition to batch-import hundreds of github repositories into gitlab. --- gitlab/tests/test_gitlab.py | 27 ++++++++++++++++ gitlab/v4/objects.py | 63 +++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/gitlab/tests/test_gitlab.py b/gitlab/tests/test_gitlab.py index 5bf373a13..3eccf6e7d 100644 --- a/gitlab/tests/test_gitlab.py +++ b/gitlab/tests/test_gitlab.py @@ -844,6 +844,33 @@ def resp_update_submodule(url, request): self.assertEqual(ret["message"], "Message") self.assertEqual(ret["id"], "ed899a2f4b50b4370feeea94676502b42383c746") + def test_import_github(self): + @urlmatch( + scheme="http", + netloc="localhost", + path="/api/v4/import/github", + method="post", + ) + def resp_import_github(url, request): + headers = {"content-type": "application/json"} + content = """{ + "id": 27, + "name": "my-repo", + "full_path": "/root/my-repo", + "full_name": "Administrator / my-repo" + }""" + content = content.encode("utf-8") + return response(200, content, headers, None, 25, request) + + with HTTMock(resp_import_github): + base_path = "/root" + name = "my-repo" + ret = self.gl.projects.import_github("githubkey", 1234, base_path, name) + self.assertIsInstance(ret, dict) + self.assertEqual(ret["name"], name) + self.assertEqual(ret["full_path"], "/".join((base_path, name))) + self.assertTrue(ret["full_name"].endswith(name)) + def _default_config(self): fd, temp_path = tempfile.mkstemp() os.write(fd, valid_config) diff --git a/gitlab/v4/objects.py b/gitlab/v4/objects.py index 2b5ed1ddc..7a1e7b84d 100644 --- a/gitlab/v4/objects.py +++ b/gitlab/v4/objects.py @@ -4744,6 +4744,69 @@ def import_project( "/projects/import", post_data=data, files=files, **kwargs ) + def import_github( + self, + personal_access_token, + repo_id, + target_namespace, + new_name=None, + timeout_override=60.0, + **kwargs + ): + """Import a project from Github to Gitlab (schedule the import) + + This method will return when an import operation has been safely queued, + or an error has occurred. After triggering an import, check the + `import_status` of the newly created project to detect when the import + operation has completed. + + NOTE: this request may take longer than most other API requests. + So this method will override the session timeout with , + which defaults to 60 seconds. + + Args: + personal_access_token (str): GitHub personal access token + repo_id (int): Github repository ID + target_namespace (str): Namespace to import repo into + new_name (str): New repo name (Optional) + timeout_override (int or float): Timeout to use for this request + **kwargs: Extra options to send to the server (e.g. sudo) + + Raises: + GitlabAuthenticationError: If authentication is not correct + GitlabListError: If the server failed to perform the request + + Returns: + dict: A representation of the import status. + + Example: + ``` + gl = gitlab.Gitlab_from_config() + print "Triggering import" + result = gl.projects.import_github(ACCESS_TOKEN, + 123456, + "my-group/my-subgroup") + project = gl.projects.get(ret['id']) + print "Waiting for import to complete" + while project.import_status == u'started': + time.sleep(1.0) + project = gl.projects.get(project.id) + print "Github import complete" + ``` + """ + data = { + "personal_access_token": personal_access_token, + "repo_id": repo_id, + "target_namespace": target_namespace, + } + if new_name: + data["new_name"] = new_name + prev_timeout = self.gitlab.timeout + self.gitlab.timeout = timeout_override + result = self.gitlab.http_post("/import/github", post_data=data, **kwargs) + self.gitlab.timeout = prev_timeout + return result + class RunnerJob(RESTObject): pass From e9a8289a381ebde7c57aa2364258d84b4771d276 Mon Sep 17 00:00:00 2001 From: Andrew Tergis Date: Tue, 10 Dec 2019 12:23:10 -0500 Subject: [PATCH 2/2] feat: allow cfg timeout to be overrided via kwargs On startup, the `timeout` parameter is loaded from config and stored on the base gitlab object instance. This instance parameter is used as the timeout for all API requests (it's passed into the `session` object when making HTTP calls). This change allows any API method to specify a `timeout` argument to `**kwargs` that will override the global timeout value. This was somewhat needed / helpful for the `import_github` method. I have also updated the docs accordingly. --- docs/api-usage.rst | 17 +++++++++++++++++ gitlab/__init__.py | 2 ++ gitlab/v4/objects.py | 26 +++++++++++++------------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/docs/api-usage.rst b/docs/api-usage.rst index fc2314ee8..d211e25ab 100644 --- a/docs/api-usage.rst +++ b/docs/api-usage.rst @@ -366,3 +366,20 @@ default an exception is raised for these errors. gl = gitlab.gitlab(url, token, api_version=4) gl.projects.list(all=True, retry_transient_errors=True) + +Timeout +------- + +python-gitlab will by default use the ``timeout`` option from it's configuration +for all requests. This is passed downwards to the ``requests`` module at the +time of making the HTTP request. However if you would like to override the +global timeout parameter for a particular call, you can provide the ``timeout`` +parameter to that API invocation: + +.. code-block:: python + + import gitlab + + gl = gitlab.gitlab(url, token, api_version=4) + gl.projects.import_github(ACCESS_TOKEN, 123456, "root", timeout=120.0) + diff --git a/gitlab/__init__.py b/gitlab/__init__.py index 03e55568d..c4460a421 100644 --- a/gitlab/__init__.py +++ b/gitlab/__init__.py @@ -491,6 +491,8 @@ def http_request( verify = opts.pop("verify") timeout = opts.pop("timeout") + # If timeout was passed into kwargs, allow it to override the default + timeout = kwargs.get("timeout", timeout) # We need to deal with json vs. data when uploading files if files: diff --git a/gitlab/v4/objects.py b/gitlab/v4/objects.py index 7a1e7b84d..65be16d1b 100644 --- a/gitlab/v4/objects.py +++ b/gitlab/v4/objects.py @@ -4745,13 +4745,7 @@ def import_project( ) def import_github( - self, - personal_access_token, - repo_id, - target_namespace, - new_name=None, - timeout_override=60.0, - **kwargs + self, personal_access_token, repo_id, target_namespace, new_name=None, **kwargs ): """Import a project from Github to Gitlab (schedule the import) @@ -4761,15 +4755,14 @@ def import_github( operation has completed. NOTE: this request may take longer than most other API requests. - So this method will override the session timeout with , - which defaults to 60 seconds. + So this method will specify a 60 second default timeout if none is specified. + A timeout can be specified via kwargs to override this functionality. Args: personal_access_token (str): GitHub personal access token repo_id (int): Github repository ID target_namespace (str): Namespace to import repo into new_name (str): New repo name (Optional) - timeout_override (int or float): Timeout to use for this request **kwargs: Extra options to send to the server (e.g. sudo) Raises: @@ -4801,10 +4794,17 @@ def import_github( } if new_name: data["new_name"] = new_name - prev_timeout = self.gitlab.timeout - self.gitlab.timeout = timeout_override + if ( + "timeout" not in kwargs + or self.gitlab.timeout is None + or self.gitlab.timeout < 60.0 + ): + # Ensure that this HTTP request has a longer-than-usual default timeout + # The base gitlab object tends to have a default that is <10 seconds, + # and this is too short for this API command, typically. + # On the order of 24 seconds has been measured on a typical gitlab instance. + kwargs["timeout"] = 60.0 result = self.gitlab.http_post("/import/github", post_data=data, **kwargs) - self.gitlab.timeout = prev_timeout return result