GeezerVison

The World Wide Web, restored to 800 pixels of glory.

GeezerVison is showing github.com — fix: make redirect and retry handling strict · flypythoncom/python@43e0f9a

File tree

Expand file treeCollapse file tree

Open diff view settings

Filter options

tests
- test_check_links.py
tools
- check_links.py

Expand file treeCollapse file tree

Open diff view settings

Collapse file

`‎tests/test_check_links.py‎`

Copy file name to clipboardExpand all lines: tests/test_check_links.py

+84-1Lines changed: 84 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -122,11 +122,61 @@ def test_head_failure_falls_back_to_streaming_get_and_confirms_404() -> None:`
`122`	`122`	`assert session.calls[1][2]["stream"] is True`
`123`	`123`
`124`	`124`
	`125`	`+def test_head_redirect_without_location_falls_back_to_get() -> None:`
	`126`	`+ head = FakeResponse(301)`
	`127`	`+ response = FakeResponse(200)`
	`128`	`+ session = FakeSession(head, response)`
	`129`	`+ checker = LinkChecker(`
	`130`	`+ guard=guard_for(), session_factory=lambda: session, workers=1, min_interval=0`
	`131`	`+ )`
	`132`	`+`
	`133`	`+ result = checker.check_one(link())`
	`134`	`+`
	`135`	`+ assert result.status == "working"`
	`136`	`+ assert result.method == "GET"`
	`137`	`+ assert head.closed is True`
	`138`	`+ assert response.closed is True`
	`139`	`+`
	`140`	`+`
	`141`	`+def test_get_redirect_without_location_is_fatal() -> None:`
	`142`	`+ head = FakeResponse(301)`
	`143`	`+ response = FakeResponse(301)`
	`144`	`+ session = FakeSession(head, response)`
	`145`	`+ checker = LinkChecker(`
	`146`	`+ guard=guard_for(), session_factory=lambda: session, workers=1, min_interval=0`
	`147`	`+ )`
	`148`	`+`
	`149`	`+ result = checker.check_one(link())`
	`150`	`+`
	`151`	`+ assert result.status == "error"`
	`152`	`+ assert "no Location" in (result.error or "")`
	`153`	`+ assert head.closed is True`
	`154`	`+ assert response.closed is True`
	`155`	`+`
	`156`	`+`
	`157`	`+def test_unsupported_redirect_status_is_fatal() -> None:`
	`158`	`+ response = FakeResponse(304)`
	`159`	`+ session = FakeSession(response)`
	`160`	`+ checker = LinkChecker(`
	`161`	`+ guard=guard_for(), session_factory=lambda: session, workers=1, min_interval=0`
	`162`	`+ )`
	`163`	`+`
	`164`	`+ result = checker.check_one(link())`
	`165`	`+`
	`166`	`+ assert result.status == "error"`
	`167`	`+ assert "unsupported redirect status 304" in (result.error or "")`
	`168`	`+ assert response.closed is True`
	`169`	`+`
	`170`	`+`
`125`	`171`	`@pytest.mark.parametrize("status_code", [403, 408, 425, 429, 500, 503])`
`126`	`172`	`def test_transient_and_access_denied_statuses_need_review(status_code) -> None:`
`127`	`173`	`session = FakeSession(FakeResponse(status_code), FakeResponse(status_code))`
`128`	`174`	`checker = LinkChecker(`
`129`		`- guard=guard_for(), session_factory=lambda: session, workers=1, min_interval=0`
	`175`	`+ guard=guard_for(),`
	`176`	`+ session_factory=lambda: session,`
	`177`	`+ workers=1,`
	`178`	`+ retries=0,`
	`179`	`+ min_interval=0,`
`130`	`180`	`)`
`131`	`181`	`assert checker.check_one(link()).status == "review"`
`132`	`182`
`@@ -347,10 +397,43 @@ def test_retry_configuration_ignores_unbounded_retry_after() -> None:`
`347`	`397`	`retry = session.get_adapter("https://").max_retries`
`348`	`398`	`assert retry.respect_retry_after_header is False`
`349`	`399`	`assert retry.backoff_max == 5.0`
	`400`	`+ assert retry.status == 0`
	`401`	`+ assert not retry.status_forcelist`
`350`	`402`	`finally:`
`351`	`403`	`session.close()`
`352`	`404`
`353`	`405`
	`406`	`+def test_status_retry_is_manual_and_closes_each_response() -> None:`
	`407`	`+ first = FakeResponse(503)`
	`408`	`+ second = FakeResponse(200)`
	`409`	`+`
	`410`	`+ class RetrySession(FakeSession):`
	`411`	`+ def __init__(self):`
	`412`	`+ super().__init__(first)`
	`413`	`+ self.responses = iter([first, second])`
	`414`	`+`
	`415`	`+ def head(self, url, **kwargs):`
	`416`	`+ self.calls.append(("HEAD", url, kwargs))`
	`417`	`+ return next(self.responses)`
	`418`	`+`
	`419`	`+ session = RetrySession()`
	`420`	`+ checker = LinkChecker(`
	`421`	`+ guard=guard_for(),`
	`422`	`+ session_factory=lambda: session,`
	`423`	`+ workers=1,`
	`424`	`+ retries=1,`
	`425`	`+ backoff_factor=0,`
	`426`	`+ min_interval=0,`
	`427`	`+ )`
	`428`	`+`
	`429`	`+ result = checker.check_one(link())`
	`430`	`+`
	`431`	`+ assert result.status == "working"`
	`432`	`+ assert len(session.calls) == 2`
	`433`	`+ assert first.closed is True`
	`434`	`+ assert second.closed is True`
	`435`	`+`
	`436`	`+`
`354`	`437`	`def test_responses_close_when_result_processing_fails(monkeypatch) -> None:`
`355`	`438`	`head = FakeResponse(404)`
`356`	`439`	`response = FakeResponse(200)`

Collapse file

`‎tools/check_links.py‎`

Copy file name to clipboardExpand all lines: tools/check_links.py

+39-5Lines changed: 39 additions & 5 deletions

Original file line number	Diff line number	Diff line change
`@@ -36,6 +36,7 @@`
`36`	`36`	`RETRY_STATUS_CODES = {408, 425, 429, 500, 502, 503, 504}`
`37`	`37`	`REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}`
`38`	`38`	`MAX_REDIRECTS = 5`
	`39`	`+MAX_BACKOFF_SECONDS = 5.0`
`39`	`40`	`KNOWN_METADATA_HOSTS = {`
`40`	`41`	`"instance-data",`
`41`	`42`	`"metadata",`
`@@ -58,6 +59,10 @@ class UnsafeTarget(ValueError):`
`58`	`59`	`"""Raised before a request can reach a non-public network target."""`
`59`	`60`
`60`	`61`
	`62`	`+class RedirectProtocolError(requests.RequestException):`
	`63`	`+ """Raised when a redirect response cannot be followed safely."""`
	`64`	`+`
	`65`	`+`
`61`	`66`	`@dataclass(frozen=True)`
`62`	`67`	`class ResolvedTarget:`
`63`	`68`	`host: str`
`@@ -198,11 +203,11 @@ def build_session(`
`198`	`203`	`total=retries,`
`199`	`204`	`connect=retries,`
`200`	`205`	`read=retries,`
`201`		`- status=retries,`
`202`	`206`	`allowed_methods=frozenset({"HEAD", "GET"}),`
`203`		`- status_forcelist=RETRY_STATUS_CODES,`
	`207`	`+ status=0,`
	`208`	`+ status_forcelist=frozenset(),`
`204`	`209`	`backoff_factor=backoff_factor,`
`205`		`- backoff_max=5.0,`
	`210`	`+ backoff_max=MAX_BACKOFF_SECONDS,`
`206`	`211`	`respect_retry_after_header=False,`
`207`	`212`	`raise_on_status=False,`
`208`	`213`	`)`
`@@ -319,6 +324,8 @@ def __init__(`
`319`	`324`	`) -> None:`
`320`	`325`	`self.timeout = timeout`
`321`	`326`	`self.workers = workers`
	`327`	`+ self.status_retries = retries`
	`328`	`+ self.backoff_factor = backoff_factor`
`322`	`329`	`self.guard = guard or SafeTargetGuard()`
`323`	`330`	`factory = session_factory or (`
`324`	`331`	`lambda: build_session(`
`@@ -333,6 +340,7 @@ def _request(`
`333`	`340`	`) -> tuple[Any, list[dict[str, Any]]]:`
`334`	`341`	`current_url = url`
`335`	`342`	`history: list[dict[str, Any]] = []`
	`343`	`+ status_attempt = 0`
`336`	`344`	`while True:`
`337`	`345`	`self.guard.resolve_url(current_url)`
`338`	`346`	`self.rate_limiter.wait(current_url)`
`@@ -346,9 +354,34 @@ def _request(`
`346`	`354`	`if method == "GET"`
`347`	`355`	`else session.head(current_url, **kwargs)`
`348`	`356`	`)`
	`357`	`+ if (`
	`358`	`+ response.status_code in RETRY_STATUS_CODES`
	`359`	`+ and status_attempt < self.status_retries`
	`360`	`+ ):`
	`361`	`+ response.close()`
	`362`	`+ delay = min(`
	`363`	`+ self.backoff_factor * (2**status_attempt), MAX_BACKOFF_SECONDS`
	`364`	`+ )`
	`365`	`+ status_attempt += 1`
	`366`	`+ if delay > 0:`
	`367`	`+ time.sleep(delay)`
	`368`	`+ continue`
	`369`	`+`
`349`	`370`	`location = response.headers.get("Location")`
`350`		`- if response.status_code not in REDIRECT_STATUS_CODES or not location:`
	`371`	`+ if not 300 <= response.status_code < 400:`
`351`	`372`	`return response, history`
	`373`	`+ if response.status_code not in REDIRECT_STATUS_CODES:`
	`374`	`+ response.close()`
	`375`	`+ raise RedirectProtocolError(`
	`376`	`+ f"unsupported redirect status {response.status_code}"`
	`377`	`+ )`
	`378`	`+ if not location:`
	`379`	`+ if method == "HEAD":`
	`380`	`+ return response, history`
	`381`	`+ response.close()`
	`382`	`+ raise RedirectProtocolError(`
	`383`	`+ f"redirect status {response.status_code} has no Location header"`
	`384`	`+ )`
`352`	`385`
`353`	`386`	`try:`
`354`	`387`	`if len(history) >= MAX_REDIRECTS:`
`@@ -371,13 +404,14 @@ def _request(`
`371`	`404`	`finally:`
`372`	`405`	`response.close()`
`373`	`406`	`current_url = next_url`
	`407`	`+ status_attempt = 0`
`374`	`408`
`375`	`409`	`def check_one(self, link: CatalogLink) -> LinkResult:`
`376`	`410`	`try:`
`377`	`411`	`session = self.sessions.get()`
`378`	`412`	`head, head_history = self._request(session, "HEAD", link.url)`
`379`	`413`	`try:`
`380`		`- if head.status_code < 400:`
	`414`	`+ if head.status_code < 300:`
`381`	`415`	`return LinkResult(`
`382`	`416`	`link.id,`
`383`	`417`	`link.path,`

0 commit comments

Comments

0 (0)

You are visitor 01402520 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.