GeezerVison

The World Wide Web, restored to 800 pixels of glory.

GeezerVison is showing github.com — feat(client): add retry handling to GraphQL client · python-gitlab/python-gitlab@8898c38

File tree

Expand file treeCollapse file tree

Open diff view settings

Filter options

gitlab
- client.py
- utils.py
requirements-lint.txt
requirements-test.txt
tests/unit
- test_graphql.py
- test_retry.py

Expand file treeCollapse file tree

Open diff view settings

Collapse file

`‎gitlab/client.py‎`

Copy file name to clipboardExpand all lines: gitlab/client.py

+29-2Lines changed: 29 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,7 @@`
`26`	`26`
`27`	`27`	`try:`
`28`	`28`	`import gql`
	`29`	`+ import gql.transport.exceptions`
`29`	`30`	`import graphql`
`30`	`31`	`import httpx`
`31`	`32`
`@@ -746,7 +747,9 @@ def http_request(`
`746`	`747`	`if 200 <= result.status_code < 300:`
`747`	`748`	`return result.response`
`748`	`749`
`749`		`- if retry.handle_retry_on_status(result):`
	`750`	`+ if retry.handle_retry_on_status(`
	`751`	`+ result.status_code, result.headers, result.reason`
	`752`	`+ ):`
`750`	`753`	`continue`
`751`	`754`
`752`	`755`	`error_message = result.content`
`@@ -1329,4 +1332,28 @@ def execute(`
`1329`	`1332`	`self, request: Union[str, graphql.Source], args: Any, *kwargs: Any`
`1330`	`1333`	`) -> Any:`
`1331`	`1334`	`parsed_document = self._gql(request)`
`1332`		`- return self._client.execute(parsed_document, args, *kwargs)`
	`1335`	`+ retry = utils.Retry(`
	`1336`	`+ max_retries=3, obey_rate_limit=True, retry_transient_errors=False`
	`1337`	`+ )`
	`1338`	`+`
	`1339`	`+ while True:`
	`1340`	`+ try:`
	`1341`	`+ result = self._client.execute(parsed_document, args, *kwargs)`
	`1342`	`+ except gql.transport.exceptions.TransportServerError as e:`
	`1343`	`+ if retry.handle_retry_on_status(`
	`1344`	`+ status_code=e.code, headers=self._transport.response_headers`
	`1345`	`+ ):`
	`1346`	`+ continue`
	`1347`	`+`
	`1348`	`+ if e.code == 401:`
	`1349`	`+ raise gitlab.exceptions.GitlabAuthenticationError(`
	`1350`	`+ response_code=e.code,`
	`1351`	`+ error_message=str(e),`
	`1352`	`+ )`
	`1353`	`+`
	`1354`	`+ raise gitlab.exceptions.GitlabHttpError(`
	`1355`	`+ response_code=e.code,`
	`1356`	`+ error_message=str(e),`
	`1357`	`+ )`
	`1358`	`+`
	`1359`	`+ return result`

Collapse file

`‎gitlab/utils.py‎`

Copy file name to clipboardExpand all lines: gitlab/utils.py

+30-13Lines changed: 30 additions & 13 deletions

Original file line number	Diff line number	Diff line change
`@@ -6,12 +6,22 @@`
`6`	`6`	`import traceback`
`7`	`7`	`import urllib.parse`
`8`	`8`	`import warnings`
`9`		`-from typing import Any, Callable, Dict, Iterator, Literal, Optional, Tuple, Type, Union`
	`9`	`+from typing import (`
	`10`	`+ Any,`
	`11`	`+ Callable,`
	`12`	`+ Dict,`
	`13`	`+ Iterator,`
	`14`	`+ Literal,`
	`15`	`+ MutableMapping,`
	`16`	`+ Optional,`
	`17`	`+ Tuple,`
	`18`	`+ Type,`
	`19`	`+ Union,`
	`20`	`+)`
`10`	`21`
`11`	`22`	`import requests`
`12`	`23`
`13`	`24`	`from gitlab import const, types`
`14`		`-from gitlab._backends import requests_backend`
`15`	`25`
`16`	`26`
`17`	`27`	`class _StdoutStream:`
`@@ -100,33 +110,40 @@ def __init__(`
`100`	`110`	`self.retry_transient_errors = retry_transient_errors`
`101`	`111`
`102`	`112`	`def _retryable_status_code(`
`103`		`- self,`
`104`		`- result: requests_backend.RequestsResponse,`
	`113`	`+ self, status_code: Optional[int], reason: str = ""`
`105`	`114`	`) -> bool:`
`106`		`- if result.status_code == 429 and self.obey_rate_limit:`
	`115`	`+ if status_code == 429 and self.obey_rate_limit:`
`107`	`116`	`return True`
`108`	`117`
`109`	`118`	`if not self.retry_transient_errors:`
`110`	`119`	`return False`
`111`		`- if result.status_code in const.RETRYABLE_TRANSIENT_ERROR_CODES:`
	`120`	`+ if status_code in const.RETRYABLE_TRANSIENT_ERROR_CODES:`
`112`	`121`	`return True`
`113`		`- if result.status_code == 409 and "Resource lock" in result.reason:`
	`122`	`+ if status_code == 409 and "Resource lock" in reason:`
`114`	`123`	`return True`
`115`	`124`
`116`	`125`	`return False`
`117`	`126`
`118`		`- def handle_retry_on_status(self, result: requests_backend.RequestsResponse) -> bool:`
`119`		`- if not self._retryable_status_code(result):`
	`127`	`+ def handle_retry_on_status(`
	`128`	`+ self,`
	`129`	`+ status_code: Optional[int],`
	`130`	`+ headers: Optional[MutableMapping[str, str]] = None,`
	`131`	`+ reason: str = "",`
	`132`	`+ ) -> bool:`
	`133`	`+ if not self._retryable_status_code(status_code, reason):`
`120`	`134`	`return False`
`121`	`135`
	`136`	`+ if headers is None:`
	`137`	`+ headers = {}`
	`138`	`+`
`122`	`139`	`# Response headers documentation:`
`123`	`140`	`# https://docs.gitlab.com/ee/user/admin_area/settings/user_and_ip_rate_limits.html#response-headers`
`124`	`141`	`if self.max_retries == -1 or self.cur_retries < self.max_retries:`
`125`	`142`	`wait_time = 2*self.cur_retries 0.1`
`126`		`- if "Retry-After" in result.headers:`
`127`		`- wait_time = int(result.headers["Retry-After"])`
`128`		`- elif "RateLimit-Reset" in result.headers:`
`129`		`- wait_time = int(result.headers["RateLimit-Reset"]) - time.time()`
	`143`	`+ if "Retry-After" in headers:`
	`144`	`+ wait_time = int(headers["Retry-After"])`
	`145`	`+ elif "RateLimit-Reset" in headers:`
	`146`	`+ wait_time = int(headers["RateLimit-Reset"]) - time.time()`
`130`	`147`	`self.cur_retries += 1`
`131`	`148`	`time.sleep(wait_time)`
`132`	`149`	`return True`

Collapse file

`‎requirements-lint.txt‎`

Copy file name to clipboardExpand all lines: requirements-lint.txt

+1Lines changed: 1 addition & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -8,6 +8,7 @@ mypy==1.11.2`
`8`	`8`	`pylint==3.2.7`
`9`	`9`	`pytest==8.3.2`
`10`	`10`	`responses==0.25.3`
	`11`	`+respx==0.21.1`
`11`	`12`	`types-PyYAML==6.0.12.20240808`
`12`	`13`	`types-requests==2.32.0.20240907`
`13`	`14`	`types-setuptools==74.1.0.20240907`

Collapse file

`‎requirements-test.txt‎`

Copy file name to clipboardExpand all lines: requirements-test.txt

+1Lines changed: 1 addition & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -7,4 +7,5 @@ pytest-github-actions-annotate-failures==0.2.0`
`7`	`7`	`pytest==8.3.2`
`8`	`8`	`PyYaml==6.0.2`
`9`	`9`	`responses==0.25.3`
	`10`	`+respx==0.21.1`
`10`	`11`	`wheel==0.44.0`

Collapse file

`‎tests/unit/test_graphql.py‎`

Copy file name to clipboardExpand all lines: tests/unit/test_graphql.py

+46Lines changed: 46 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,15 @@`
	`1`	`+import httpx`
`1`	`2`	`import pytest`
	`3`	`+import respx`
`2`	`4`
`3`	`5`	`import gitlab`
`4`	`6`
`5`	`7`
	`8`	`+@pytest.fixture`
	`9`	`+def gl_gql() -> gitlab.GraphQL:`
	`10`	`+ return gitlab.GraphQL("https://gitlab.example.com")`
	`11`	`+`
	`12`	`+`
`6`	`13`	`def test_import_error_includes_message(monkeypatch: pytest.MonkeyPatch):`
`7`	`14`	`monkeypatch.setattr(gitlab.client, "_GQL_INSTALLED", False)`
`8`	`15`	`with pytest.raises(ImportError, match="GraphQL client could not be initialized"):`
`@@ -12,3 +19,42 @@ def test_import_error_includes_message(monkeypatch: pytest.MonkeyPatch):`
`12`	`19`	`def test_graphql_as_context_manager_exits():`
`13`	`20`	`with gitlab.GraphQL() as gl:`
`14`	`21`	`assert isinstance(gl, gitlab.GraphQL)`
	`22`	`+`
	`23`	`+`
	`24`	`+def test_graphql_retries_on_429_response(`
	`25`	`+ gl_gql: gitlab.GraphQL, respx_mock: respx.MockRouter`
	`26`	`+):`
	`27`	`+ url = "https://gitlab.example.com/api/graphql"`
	`28`	`+ responses = [`
	`29`	`+ httpx.Response(429, headers={"retry-after": "1"}),`
	`30`	`+ httpx.Response(`
	`31`	`+ 200, json={"data": {"currentUser": {"id": "gid://gitlab/User/1"}}}`
	`32`	`+ ),`
	`33`	`+ ]`
	`34`	`+ respx_mock.post(url).mock(side_effect=responses)`
	`35`	`+ gl_gql.execute("query {currentUser {id}}")`
	`36`	`+`
	`37`	`+`
	`38`	`+def test_graphql_raises_when_max_retries_exceeded(`
	`39`	`+ gl_gql: gitlab.GraphQL, respx_mock: respx.MockRouter`
	`40`	`+):`
	`41`	`+ url = "https://gitlab.example.com/api/graphql"`
	`42`	`+ responses = [`
	`43`	`+ httpx.Response(502),`
	`44`	`+ httpx.Response(502),`
	`45`	`+ httpx.Response(502),`
	`46`	`+ httpx.Response(502),`
	`47`	`+ httpx.Response(502),`
	`48`	`+ ]`
	`49`	`+ respx_mock.post(url).mock(side_effect=responses)`
	`50`	`+ with pytest.raises(gitlab.GitlabHttpError):`
	`51`	`+ gl_gql.execute("query {currentUser {id}}")`
	`52`	`+`
	`53`	`+`
	`54`	`+def test_graphql_raises_on_401_response(`
	`55`	`+ gl_gql: gitlab.GraphQL, respx_mock: respx.MockRouter`
	`56`	`+):`
	`57`	`+ url = "https://gitlab.example.com/api/graphql"`
	`58`	`+ respx_mock.post(url).mock(return_value=httpx.Response(401))`
	`59`	`+ with pytest.raises(gitlab.GitlabAuthenticationError):`
	`60`	`+ gl_gql.execute("query {currentUser {id}}")`

Collapse file

`‎tests/unit/test_retry.py‎`

Copy file name to clipboardExpand all lines: tests/unit/test_retry.py

+6-23Lines changed: 6 additions & 23 deletions

Original file line number	Diff line number	Diff line change
`@@ -2,34 +2,24 @@`
`2`	`2`	`from unittest import mock`
`3`	`3`
`4`	`4`	`import pytest`
`5`		`-import requests`
`6`	`5`
`7`	`6`	`from gitlab import utils`
`8`		`-from gitlab._backends import requests_backend`
`9`	`7`
`10`	`8`
`11`	`9`	`def test_handle_retry_on_status_ignores_unknown_status_code():`
`12`	`10`	`retry = utils.Retry(max_retries=1, retry_transient_errors=True)`
`13`		`- response = requests.Response()`
`14`		`- response.status_code = 418`
`15`		`- backend_response = requests_backend.RequestsResponse(response)`
`16`		`-`
`17`		`- assert retry.handle_retry_on_status(backend_response) is False`
	`11`	`+ assert retry.handle_retry_on_status(418) is False`
`18`	`12`
`19`	`13`
`20`	`14`	`def test_handle_retry_on_status_accepts_retry_after_header(`
`21`	`15`	`monkeypatch: pytest.MonkeyPatch,`
`22`	`16`	`):`
`23`	`17`	`mock_sleep = mock.Mock()`
`24`	`18`	`monkeypatch.setattr(time, "sleep", mock_sleep)`
`25`		`-`
`26`	`19`	`retry = utils.Retry(max_retries=1)`
`27`		`- response = requests.Response()`
`28`		`- response.status_code = 429`
`29`		`- response.headers["Retry-After"] = "1"`
`30`		`- backend_response = requests_backend.RequestsResponse(response)`
	`20`	`+ headers = {"Retry-After": "1"}`
`31`	`21`
`32`		`- assert retry.handle_retry_on_status(backend_response) is True`
	`22`	`+ assert retry.handle_retry_on_status(429, headers=headers) is True`
`33`	`23`	`assert isinstance(mock_sleep.call_args[0][0], int)`
`34`	`24`
`35`	`25`
`@@ -40,19 +30,12 @@ def test_handle_retry_on_status_accepts_ratelimit_reset_header(`
`40`	`30`	`monkeypatch.setattr(time, "sleep", mock_sleep)`
`41`	`31`
`42`	`32`	`retry = utils.Retry(max_retries=1)`
`43`		`- response = requests.Response()`
`44`		`- response.status_code = 429`
`45`		`- response.headers["RateLimit-Reset"] = str(int(time.time() + 1))`
`46`		`- backend_response = requests_backend.RequestsResponse(response)`
	`33`	`+ headers = {"RateLimit-Reset": str(int(time.time() + 1))}`
`47`	`34`
`48`		`- assert retry.handle_retry_on_status(backend_response) is True`
	`35`	`+ assert retry.handle_retry_on_status(429, headers=headers) is True`
`49`	`36`	`assert isinstance(mock_sleep.call_args[0][0], float)`
`50`	`37`
`51`	`38`
`52`	`39`	`def test_handle_retry_on_status_returns_false_when_max_retries_reached():`
`53`	`40`	`retry = utils.Retry(max_retries=0)`
`54`		`- response = requests.Response()`
`55`		`- response.status_code = 429`
`56`		`- backend_response = requests_backend.RequestsResponse(response)`
`57`		`-`
`58`		`- assert retry.handle_retry_on_status(backend_response) is False`
	`41`	`+ assert retry.handle_retry_on_status(429) is False`

0 commit comments

Comments

0 (0)

You are visitor 01402520 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.