|
2 | 2 | import threading |
3 | 3 | from datetime import datetime, timedelta |
4 | 4 | from json import dumps |
5 | | -from typing import Any, Iterable, List, Mapping |
| 5 | +from typing import Any, Dict, Generator, Iterable, List, Mapping, Optional, Tuple |
6 | 6 |
|
7 | 7 | import pkg_resources |
8 | 8 | import pyramid.httpexceptions as exc |
|
13 | 13 | from lxml import etree |
14 | 14 | from pyramid.config import Configurator |
15 | 15 | from pyramid.events import NewRequest |
| 16 | +from pyramid.request import Request |
16 | 17 | from pyramid.response import Response |
17 | 18 | from six import b |
18 | 19 | from six.moves.urllib_parse import quote_plus |
|
22 | 23 | from pyff.logs import get_log |
23 | 24 | from pyff.pipes import plumbing |
24 | 25 | from pyff.repo import MDRepository |
25 | | -from pyff.resource import Resource, ResourceInfo |
| 26 | +from pyff.resource import Resource |
26 | 27 | from pyff.samlmd import entity_display_name |
27 | | -from pyff.utils import b2u, dumptree, duration2timedelta, hash_id, json_serializer, utc_now |
| 28 | +from pyff.utils import b2u, dumptree, hash_id, json_serializer, utc_now |
28 | 29 |
|
29 | 30 | log = get_log(__name__) |
30 | 31 |
|
31 | 32 |
|
32 | 33 | class NoCache(object): |
33 | | - def __init__(self): |
| 34 | + """ Dummy implementation for when caching isn't enabled """ |
| 35 | + |
| 36 | + def __init__(self) -> None: |
34 | 37 | pass |
35 | 38 |
|
36 | | - def __getitem__(self, item): |
| 39 | + def __getitem__(self, item: Any) -> None: |
37 | 40 | return None |
38 | 41 |
|
39 | | - def __setitem__(self, instance, value): |
| 42 | + def __setitem__(self, instance: Any, value: Any) -> Any: |
40 | 43 | return value |
41 | 44 |
|
42 | 45 |
|
43 | | -def robots_handler(request): |
| 46 | +def robots_handler(request: Request) -> Response: |
44 | 47 | """ |
45 | | - Impelements robots.txt |
| 48 | + Implements robots.txt |
46 | 49 |
|
47 | 50 | :param request: the HTTP request |
48 | 51 | :return: robots.txt |
@@ -55,7 +58,7 @@ def robots_handler(request): |
55 | 58 | ) |
56 | 59 |
|
57 | 60 |
|
58 | | -def status_handler(request): |
| 61 | +def status_handler(request: Request) -> Response: |
59 | 62 | """ |
60 | 63 | Implements the /api/status endpoint |
61 | 64 |
|
@@ -80,34 +83,38 @@ def status_handler(request): |
80 | 83 |
|
81 | 84 |
|
82 | 85 | class MediaAccept(object): |
83 | | - def __init__(self, accept): |
| 86 | + def __init__(self, accept: str): |
84 | 87 | self._type = AcceptableType(accept) |
85 | 88 |
|
86 | | - def has_key(self, key): |
| 89 | + def has_key(self, key: Any) -> bool: # Literal[True]: |
87 | 90 | return True |
88 | 91 |
|
89 | | - def get(self, item): |
| 92 | + def get(self, item: Any) -> Any: |
90 | 93 | return self._type.matches(item) |
91 | 94 |
|
92 | | - def __contains__(self, item): |
| 95 | + def __contains__(self, item: Any) -> Any: |
93 | 96 | return self._type.matches(item) |
94 | 97 |
|
95 | | - def __str__(self): |
| 98 | + def __str__(self) -> str: |
96 | 99 | return str(self._type) |
97 | 100 |
|
98 | 101 |
|
99 | 102 | xml_types = ('text/xml', 'application/xml', 'application/samlmetadata+xml') |
100 | 103 |
|
101 | 104 |
|
102 | | -def _is_xml_type(accepter): |
| 105 | +def _is_xml_type(accepter: MediaAccept) -> bool: |
103 | 106 | return any([x in accepter for x in xml_types]) |
104 | 107 |
|
105 | 108 |
|
106 | | -def _is_xml(data): |
| 109 | +def _is_xml(data: Any) -> bool: |
107 | 110 | return isinstance(data, (etree._Element, etree._ElementTree)) |
108 | 111 |
|
109 | 112 |
|
110 | | -def _fmt(data, accepter): |
| 113 | +def _fmt(data: Any, accepter: MediaAccept) -> Tuple[str, str]: |
| 114 | + """ |
| 115 | + Format data according to the accepted content type of the requester. |
| 116 | + Return data as string (either XML or json) and a content-type. |
| 117 | + """ |
111 | 118 | if data is None or len(data) == 0: |
112 | 119 | return "", 'text/plain' |
113 | 120 | if _is_xml(data) and _is_xml_type(accepter): |
@@ -127,7 +134,7 @@ def call(entry: str) -> None: |
127 | 134 | return None |
128 | 135 |
|
129 | 136 |
|
130 | | -def request_handler(request): |
| 137 | +def request_handler(request: Request) -> Response: |
131 | 138 | """ |
132 | 139 | The main GET request handler for pyFF. Implements caching and forwards the request to process_handler |
133 | 140 |
|
@@ -146,7 +153,7 @@ def request_handler(request): |
146 | 153 | return r |
147 | 154 |
|
148 | 155 |
|
149 | | -def process_handler(request): |
| 156 | +def process_handler(request: Request) -> Response: |
150 | 157 | """ |
151 | 158 | The main request handler for pyFF. Implements API call hooks and content negotiation. |
152 | 159 |
|
@@ -155,7 +162,8 @@ def process_handler(request): |
155 | 162 | """ |
156 | 163 | _ctypes = {'xml': 'application/samlmetadata+xml;application/xml;text/xml', 'json': 'application/json'} |
157 | 164 |
|
158 | | - def _d(x, do_split=True): |
| 165 | + def _d(x: Optional[str], do_split: bool = True) -> Tuple[Optional[str], Optional[str]]: |
| 166 | + """ Split a path into a base component and an extension. """ |
159 | 167 | if x is not None: |
160 | 168 | x = x.strip() |
161 | 169 |
|
@@ -170,7 +178,7 @@ def _d(x, do_split=True): |
170 | 178 |
|
171 | 179 | return x, None |
172 | 180 |
|
173 | | - log.debug(request) |
| 181 | + log.debug(f'Processing request: {request}') |
174 | 182 |
|
175 | 183 | if request.matchdict is None: |
176 | 184 | raise exc.exception_response(400) |
@@ -182,18 +190,18 @@ def _d(x, do_split=True): |
182 | 190 | pass |
183 | 191 |
|
184 | 192 | entry = request.matchdict.get('entry', 'request') |
185 | | - path = list(request.matchdict.get('path', [])) |
| 193 | + path_elem = list(request.matchdict.get('path', [])) |
186 | 194 | match = request.params.get('q', request.params.get('query', None)) |
187 | 195 |
|
188 | 196 | # Enable matching on scope. |
189 | 197 | match = match.split('@').pop() if match and not match.endswith('@') else match |
190 | 198 | log.debug("match={}".format(match)) |
191 | 199 |
|
192 | | - if 0 == len(path): |
193 | | - path = ['entities'] |
| 200 | + if not path_elem: |
| 201 | + path_elem = ['entities'] |
194 | 202 |
|
195 | | - alias = path.pop(0) |
196 | | - path = '/'.join(path) |
| 203 | + alias = path_elem.pop(0) |
| 204 | + path = '/'.join(path_elem) |
197 | 205 |
|
198 | 206 | # Ugly workaround bc WSGI drops double-slashes. |
199 | 207 | path = path.replace(':/', '://') |
@@ -226,23 +234,31 @@ def _d(x, do_split=True): |
226 | 234 | accept = str(request.accept).split(',')[0] |
227 | 235 | valid_accept = accept and not ('application/*' in accept or 'text/*' in accept or '*/*' in accept) |
228 | 236 |
|
229 | | - path_no_extension, extension = _d(path, True) |
230 | | - accept_from_extension = _ctypes.get(extension, accept) |
| 237 | + new_path: Optional[str] = path |
| 238 | + path_no_extension, extension = _d(new_path, True) |
| 239 | + accept_from_extension = accept |
| 240 | + if extension: |
| 241 | + accept_from_extension = _ctypes.get(extension, accept) |
231 | 242 |
|
232 | 243 | if policy == 'extension': |
233 | | - path = path_no_extension |
| 244 | + new_path = path_no_extension |
234 | 245 | if not valid_accept: |
235 | 246 | accept = accept_from_extension |
236 | 247 | elif policy == 'adaptive': |
237 | 248 | if not valid_accept: |
238 | | - path = path_no_extension |
| 249 | + new_path = path_no_extension |
239 | 250 | accept = accept_from_extension |
240 | 251 |
|
241 | | - if pfx and path: |
242 | | - q = "{%s}%s" % (pfx, path) |
243 | | - path = "/%s/%s" % (alias, path) |
| 252 | + if not accept: |
| 253 | + log.warning('Could not determine accepted response type') |
| 254 | + raise exc.exception_response(400) |
| 255 | + |
| 256 | + q: Optional[str] |
| 257 | + if pfx and new_path: |
| 258 | + q = f'{{{pfx}}}{new_path}' |
| 259 | + new_path = f'/{alias}/{new_path}' |
244 | 260 | else: |
245 | | - q = path |
| 261 | + q = new_path |
246 | 262 |
|
247 | 263 | try: |
248 | 264 | accepter = MediaAccept(accept) |
@@ -254,18 +270,19 @@ def _d(x, do_split=True): |
254 | 270 | 'url': request.current_route_url(), |
255 | 271 | 'select': q, |
256 | 272 | 'match': match.lower() if match else match, |
257 | | - 'path': path, |
| 273 | + 'path': new_path, |
258 | 274 | 'stats': {}, |
259 | 275 | } |
260 | 276 |
|
261 | 277 | r = p.process(request.registry.md, state=state, raise_exceptions=True, scheduler=request.registry.scheduler) |
262 | | - log.debug(r) |
| 278 | + log.debug(f'Plumbing process result: {r}') |
263 | 279 | if r is None: |
264 | 280 | r = [] |
265 | 281 |
|
266 | 282 | response = Response() |
267 | | - response.headers.update(state.get('headers', {})) |
268 | | - ctype = state.get('headers').get('Content-Type', None) |
| 283 | + _headers = state.get('headers', {}) |
| 284 | + response.headers.update(_headers) |
| 285 | + ctype = _headers.get('Content-Type', None) |
269 | 286 | if not ctype: |
270 | 287 | r, t = _fmt(r, accepter) |
271 | 288 | ctype = t |
@@ -280,20 +297,20 @@ def _d(x, do_split=True): |
280 | 297 | import traceback |
281 | 298 |
|
282 | 299 | log.debug(traceback.format_exc()) |
283 | | - log.warning(ex) |
| 300 | + log.warning(f'Exception from processing pipeline: {ex}') |
284 | 301 | raise exc.exception_response(409) |
285 | 302 | except BaseException as ex: |
286 | 303 | import traceback |
287 | 304 |
|
288 | 305 | log.debug(traceback.format_exc()) |
289 | | - log.error(ex) |
| 306 | + log.error(f'Exception from processing pipeline: {ex}') |
290 | 307 | raise exc.exception_response(500) |
291 | 308 |
|
292 | 309 | if request.method == 'GET': |
293 | 310 | raise exc.exception_response(404) |
294 | 311 |
|
295 | 312 |
|
296 | | -def webfinger_handler(request): |
| 313 | +def webfinger_handler(request: Request) -> Response: |
297 | 314 | """An implementation the webfinger protocol |
298 | 315 | (http://tools.ietf.org/html/draft-ietf-appsawg-webfinger-12) |
299 | 316 | in order to provide information about up and downstream metadata available at |
@@ -324,7 +341,7 @@ def webfinger_handler(request): |
324 | 341 | "subject": "http://reep.refeds.org:8080" |
325 | 342 | } |
326 | 343 |
|
327 | | - Depending on which version of pyFF your're running and the configuration you |
| 344 | + Depending on which version of pyFF you're running and the configuration you |
328 | 345 | may also see downstream metadata listed using the 'role' attribute to the link |
329 | 346 | elements. |
330 | 347 | """ |
@@ -335,11 +352,11 @@ def webfinger_handler(request): |
335 | 352 | if resource is None: |
336 | 353 | resource = request.host_url |
337 | 354 |
|
338 | | - jrd = dict() |
339 | | - dt = datetime.now() + duration2timedelta("PT1H") |
| 355 | + jrd: Dict[str, Any] = dict() |
| 356 | + dt = datetime.now() + timedelta(hours=1) |
340 | 357 | jrd['expires'] = dt.isoformat() |
341 | 358 | jrd['subject'] = request.host_url |
342 | | - links = list() |
| 359 | + links: List[Dict[str, Any]] = list() |
343 | 360 | jrd['links'] = links |
344 | 361 |
|
345 | 362 | _dflt_rels = { |
@@ -352,7 +369,7 @@ def webfinger_handler(request): |
352 | 369 | else: |
353 | 370 | rel = [rel] |
354 | 371 |
|
355 | | - def _links(url, title=None): |
| 372 | + def _links(url: str, title: Any = None) -> None: |
356 | 373 | if url.startswith('/'): |
357 | 374 | url = url.lstrip('/') |
358 | 375 | for r in rel: |
@@ -381,7 +398,7 @@ def _links(url, title=None): |
381 | 398 | return response |
382 | 399 |
|
383 | 400 |
|
384 | | -def resources_handler(request): |
| 401 | +def resources_handler(request: Request) -> Response: |
385 | 402 | """ |
386 | 403 | Implements the /api/resources endpoint |
387 | 404 |
|
@@ -409,7 +426,7 @@ def _info(r: Resource) -> Mapping[str, Any]: |
409 | 426 | return response |
410 | 427 |
|
411 | 428 |
|
412 | | -def pipeline_handler(request): |
| 429 | +def pipeline_handler(request: Request) -> Response: |
413 | 430 | """ |
414 | 431 | Implements the /api/pipeline endpoint |
415 | 432 |
|
@@ -422,7 +439,7 @@ def pipeline_handler(request): |
422 | 439 | return response |
423 | 440 |
|
424 | 441 |
|
425 | | -def search_handler(request): |
| 442 | +def search_handler(request: Request) -> Response: |
426 | 443 | """ |
427 | 444 | Implements the /api/search endpoint |
428 | 445 |
|
@@ -438,7 +455,7 @@ def search_handler(request): |
438 | 455 | log.debug("match={}".format(match)) |
439 | 456 | store = request.registry.md.store |
440 | 457 |
|
441 | | - def _response(): |
| 458 | + def _response() -> Generator[bytes, bytes, None]: |
442 | 459 | yield b('[') |
443 | 460 | in_loop = False |
444 | 461 | entities = store.search(query=match.lower(), entity_filter=entity_filter) |
@@ -454,8 +471,8 @@ def _response(): |
454 | 471 | return response |
455 | 472 |
|
456 | 473 |
|
457 | | -def add_cors_headers_response_callback(event): |
458 | | - def cors_headers(request, response): |
| 474 | +def add_cors_headers_response_callback(event: NewRequest) -> None: |
| 475 | + def cors_headers(request: Request, response: Response) -> None: |
459 | 476 | response.headers.update( |
460 | 477 | { |
461 | 478 | 'Access-Control-Allow-Origin': '*', |
@@ -469,7 +486,7 @@ def cors_headers(request, response): |
469 | 486 | event.request.add_response_callback(cors_headers) |
470 | 487 |
|
471 | 488 |
|
472 | | -def launch_memory_usage_server(port=9002): |
| 489 | +def launch_memory_usage_server(port: int = 9002) -> None: |
473 | 490 | import cherrypy |
474 | 491 | import dowser |
475 | 492 |
|
@@ -479,7 +496,7 @@ def launch_memory_usage_server(port=9002): |
479 | 496 | cherrypy.engine.start() |
480 | 497 |
|
481 | 498 |
|
482 | | -def mkapp(*args, **kwargs): |
| 499 | +def mkapp(*args: Any, **kwargs: Any) -> Any: |
483 | 500 | md = kwargs.pop('md', None) |
484 | 501 | if md is None: |
485 | 502 | md = MDRepository() |
@@ -501,7 +518,9 @@ def mkapp(*args, **kwargs): |
501 | 518 | for mn in config.modules: |
502 | 519 | importlib.import_module(mn) |
503 | 520 |
|
504 | | - pipeline = args or None |
| 521 | + pipeline = None |
| 522 | + if args: |
| 523 | + pipeline = list(args) |
505 | 524 | if pipeline is None and config.pipeline: |
506 | 525 | pipeline = [config.pipeline] |
507 | 526 |
|
|