GeezerVison is showing github.com — gh-78416: Support required keyword-only arguments in PyArg_ParseTuple… · serhiy-storchaka/cpython@625e7b2
File treeExpand file treeCollapse file tree Open diff view settings Expand file treeCollapse file tree Open diff view settings Collapse file Copy file name to clipboardExpand all lines: Doc/c-api/arg.rst +12-4Lines changed: 12 additions & 4 deletions - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
@@ -402,18 +402,26 @@ inside nested parentheses. They are: | 402 | 402 | For example, the format string ``"OO|OO"`` corresponds to the Python | 403 | 403 | signature ``f(a, b, c=None, d=None)``. | 404 | 404 |
| | 405 | + :c:func:`PyArg_ParseTupleAndKeywords` only: | | 406 | + after ``$`` it indicates that the remaining keyword-only arguments are optional. | | 407 | + | 405 | 408 | ``$`` | 406 | 409 | :c:func:`PyArg_ParseTupleAndKeywords` only: | 407 | 410 | Indicates that the remaining arguments in the Python argument list are | 408 | 411 | keyword-only. | 409 | | - They are optional if ``|`` was specified before ``$``, and required otherwise. | 410 | | - ``|`` cannot be specified after ``$``. | 411 | | - For example, the format string ``"O|O$O"`` corresponds to the Python | 412 | | - signature ``f(a, b=None, *, c=None)``, | | 412 | + They are required until ``|``, if it follows. | | 413 | + Otherwise they are optional if ``|`` was specified before ``$``, | | 414 | + and required if it was not. | | 415 | + For example, the format string ``"O|O$O|O"`` corresponds to the Python | | 416 | + signature ``f(a, b=None, *, c, d=None)``, | | 417 | + the format string ``"O|O$O"`` corresponds to ``f(a, b=None, *, c=None)``, | 413 | 418 | and the format string ``"OO$OO"`` corresponds to ``f(a, b, *, c, d)``. | 414 | 419 |
| 415 | 420 | .. versionadded:: 3.3 | 416 | 421 |
| | 422 | + .. versionchanged:: next | | 423 | + ``|`` can be specified after ``$``. | | 424 | + | 417 | 425 | ``:`` | 418 | 426 | The list of format units ends here; the string after the colon is used as the | 419 | 427 | function name in error messages (the "associated value" of the exception that | |
Collapse file Copy file name to clipboardExpand all lines: Doc/whatsnew/3.16.rst +7-1Lines changed: 7 additions & 1 deletion - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
@@ -845,7 +845,13 @@ C API changes | 845 | 845 | New features | 846 | 846 | ------------ | 847 | 847 |
| 848 | | -* TODO | | 848 | +* :c:func:`PyArg_ParseTupleAndKeywords` now supports required keyword-only | | 849 | + arguments mixed with optional arguments. | | 850 | + ``|`` can now be specified after ``$`` to end the required keyword-only | | 851 | + arguments. | | 852 | + For example, the format string ``"O|O$O|O"`` corresponds to the Python | | 853 | + signature ``f(a, b=None, *, c, d=None)``. | | 854 | + (Contributed by Serhiy Storchaka in :gh:`78416`.) | 849 | 855 |
| 850 | 856 | Porting to Python 3.16 | 851 | 857 | ---------------------- | |
Collapse file Copy file name to clipboardExpand all lines: Include/cpython/modsupport.h +1Lines changed: 1 addition & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -31,6 +31,7 @@ typedef struct _PyArg_Parser { | 31 | 31 | int pos; /* number of positional-only arguments */ | 32 | 32 | int min; /* minimal number of arguments */ | 33 | 33 | int max; /* maximal number of positional arguments */ | | 34 | + int minkw; /* index of the first optional keyword-only argument */ | 34 | 35 | PyObject *kwtuple; /* tuple of keyword parameter names */ | 35 | 36 | struct _PyArg_Parser *next; | 36 | 37 | } _PyArg_Parser; | |
Collapse file Copy file name to clipboardExpand all lines: Lib/test/test_capi/test_getargs.py +74-2Lines changed: 74 additions & 2 deletions | Original file line number | Diff line number | Diff line change |
|---|
| 1 | 1 | import string | 2 | 2 | import sys | 3 | 3 | import unittest | | 4 | +from functools import partial | 4 | 5 | from test import support | 5 | 6 | from test.support import import_helper | 6 | 7 | from test.support import script_helper | @@ -825,6 +826,79 @@ def __hash__(self): | 825 | 826 | getargs_keyword_only(1, 2, **{BadStr("monster"): 666}) | 826 | 827 |
| 827 | 828 |
| | 829 | +class RequiredKeywordOnly_TestCase(unittest.TestCase): | | 830 | + # The keyword-only arguments after "$" are required until "|", | | 831 | + # so that required and optional ones can be mixed. | | 832 | + | | 833 | + def parse(self, format, args, kwargs, keywords=('a', 'b', 'c', 'd')): | | 834 | + return _testcapi.parse_tuple_and_keywords(args, kwargs, | | 835 | + format, list(keywords)) | | 836 | + | | 837 | + def test_required_after_optional(self): | | 838 | + # f(a, b=None, *, c, d=None) | | 839 | + parse = partial(self.parse, "O|O$O|O") | | 840 | + self.assertEqual(parse((1, 2), {'c': 3, 'd': 4}), (1, 2, 3, 4)) | | 841 | + self.assertEqual(parse((1,), {'c': 3}), (1, None, 3, None)) | | 842 | + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): | | 843 | + parse((1,), {}) | | 844 | + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): | | 845 | + parse((1,), {'d': 4}) | | 846 | + with self.assertRaisesRegex(TypeError, "at most 2 positional"): | | 847 | + parse((1, 2, 3), {'c': 3}) | | 848 | + | | 849 | + def test_all_keyword_only_required(self): | | 850 | + # f(a, b=None, *, c, d) | | 851 | + parse = partial(self.parse, "O|O$OO|") | | 852 | + self.assertEqual(parse((1,), {'c': 3, 'd': 4}), (1, None, 3, 4)) | | 853 | + with self.assertRaisesRegex(TypeError, "missing required argument 'd'"): | | 854 | + parse((1,), {'c': 3}) | | 855 | + | | 856 | + def test_all_positional_required(self): | | 857 | + # f(a, b, *, c, d=None) | | 858 | + parse = partial(self.parse, "OO$O|O") | | 859 | + self.assertEqual(parse((1, 2), {'c': 3}), (1, 2, 3, None)) | | 860 | + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): | | 861 | + parse((1, 2), {}) | | 862 | + | | 863 | + def test_cached_parser(self): | | 864 | + # The same format, parsed once and cached in a _PyArg_Parser. | | 865 | + f = _testcapi.getargs_fast_required_kwonly | | 866 | + self.assertEqual(f(1, 2, c=3, d=4), (1, 2, 3, 4)) | | 867 | + self.assertEqual(f(1, c=3), (1, None, 3, None)) | | 868 | + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): | | 869 | + f(1) | | 870 | + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): | | 871 | + f(1, d=4) | | 872 | + | | 873 | + def test_invalid_format(self): | | 874 | + for format, msg in ( | | 875 | + ("O$O$O", r"\$ specified twice"), | | 876 | + ("O$O|O|O", r"\| specified twice"), | | 877 | + ): | | 878 | + with self.subTest(format=format): | | 879 | + n = format.count('O') | | 880 | + npos = len(format) - len(format.lstrip('O')) | | 881 | + args = tuple(range(npos)) | | 882 | + kwargs = {'abcd'[i]: i for i in range(npos, n)} | | 883 | + with self.assertRaisesRegex(SystemError, msg): | | 884 | + self.parse(format, args, kwargs, 'abcd'[:n]) | | 885 | + | | 886 | + def test_unchanged_meaning_of_dollar(self): | | 887 | + # Without "|" after it, "$" still inherits the state of the | | 888 | + # positional arguments. | | 889 | + parse = partial(self.parse, "O|O$O", keywords=('a', 'b', 'c')) | | 890 | + self.assertEqual(parse((1,), {}), (1, None, None)) | | 891 | + parse = partial(self.parse, "OO$O", keywords=('a', 'b', 'c')) | | 892 | + self.assertEqual(parse((1, 2), {'c': 3}), (1, 2, 3)) | | 893 | + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): | | 894 | + parse((1, 2), {}) | | 895 | + # The same with a cached parser. | | 896 | + f = _testcapi.getargs_fast_kwonly | | 897 | + self.assertEqual(f(1, 2, c=3, d=4), (1, 2, 3, 4)) | | 898 | + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): | | 899 | + f(1, 2) | | 900 | + | | 901 | + | 828 | 902 | class PositionalOnlyAndKeywords_TestCase(unittest.TestCase): | 829 | 903 | from _testcapi import getargs_positional_only_and_keywords as getargs | 830 | 904 |
| @@ -1275,8 +1349,6 @@ def test_bad_use(self): | 1275 | 1349 | (), {'a': 1, 'b': 2}, '$O$O', ['a', 'b']) | 1276 | 1350 | self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, | 1277 | 1351 | (), {'a': 1}, '$|O', ['a']) | 1278 | | - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, | 1279 | | - (), {'a': 1, 'b': 2}, '$O|O', ['a', 'b']) | 1280 | 1352 | self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, | 1281 | 1353 | (1,), {}, '|O', ['a', 'b']) | 1282 | 1354 | self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, | |
Collapse file Copy file name to clipboardExpand all lines: Lib/test/test_clinic.py +89Lines changed: 89 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -2207,6 +2207,36 @@ def test_depr_slash_duplicate2(self): | 2207 | 2207 | err = "Function 'bar': '/ [from 3.14]' must precede '/ [from 3.15]'" | 2208 | 2208 | self.expect_failure(block, err, lineno=5) | 2209 | 2209 |
| | 2210 | + def test_required_keyword_only_after_optional(self): | | 2211 | + function = self.parse_function(""" | | 2212 | + module foo | | 2213 | + foo.bar | | 2214 | + a: int | | 2215 | + b: int = 0 | | 2216 | + * | | 2217 | + c: int | | 2218 | + d: int = 0 | | 2219 | + Docstring. | | 2220 | + """) | | 2221 | + _, a, b, c, d = function.parameters.values() | | 2222 | + self.assertFalse(a.converter.is_optional()) | | 2223 | + self.assertTrue(b.converter.is_optional()) | | 2224 | + self.assertFalse(c.converter.is_optional()) | | 2225 | + self.assertTrue(d.converter.is_optional()) | | 2226 | + | | 2227 | + def test_optional_before_required_keyword_only(self): | | 2228 | + block = """ | | 2229 | + module foo | | 2230 | + foo.bar | | 2231 | + * | | 2232 | + a: int = 0 | | 2233 | + b: int | | 2234 | + Docstring. | | 2235 | + """ | | 2236 | + err = ("Can't have a parameter without a default ('b') " | | 2237 | + "after a parameter with a default!") | | 2238 | + self.expect_failure(block, err, lineno=4) | | 2239 | + | 2210 | 2240 | def test_single_slash(self): | 2211 | 2241 | block = """ | 2212 | 2242 | module foo | @@ -4664,6 +4694,53 @@ def test_limited_capi_float(self): | 4664 | 4694 | self.assertIn("float f;", generated) | 4665 | 4695 | self.assertIn("f = (float) PyFloat_AsDouble", generated) | 4666 | 4696 |
| | 4697 | + def test_limited_capi_required_keyword_only(self): | | 4698 | + block = self.wrap_clinic_input(""" | | 4699 | + func | | 4700 | + a: object | | 4701 | + b: object = None | | 4702 | + * | | 4703 | + c: object | | 4704 | + d: object = None | | 4705 | + """) | | 4706 | + generated = self.clinic.parse(block) | | 4707 | + # "|" after "$" ends the required keyword-only parameters. | | 4708 | + self.assertIn('"O|O$O|O:func"', generated) | | 4709 | + | | 4710 | + def test_limited_capi_optional_after_required_keyword_only(self): | | 4711 | + block = self.wrap_clinic_input(""" | | 4712 | + func | | 4713 | + a: object | | 4714 | + * | | 4715 | + b: object | | 4716 | + c: object = None | | 4717 | + """) | | 4718 | + generated = self.clinic.parse(block) | | 4719 | + self.assertIn('"O$O|O:func"', generated) | | 4720 | + | | 4721 | + def test_limited_capi_no_optional_keyword_only(self): | | 4722 | + block = self.wrap_clinic_input(""" | | 4723 | + func | | 4724 | + a: object | | 4725 | + b: object = None | | 4726 | + * | | 4727 | + c: object | | 4728 | + """) | | 4729 | + generated = self.clinic.parse(block) | | 4730 | + # A trailing "|" is needed to end the required keyword-only parameters. | | 4731 | + self.assertIn('"O|O$O|:func"', generated) | | 4732 | + | | 4733 | + def test_limited_capi_keyword_only(self): | | 4734 | + # A lone "$" is enough if it can express the signature. | | 4735 | + block = self.wrap_clinic_input(""" | | 4736 | + func | | 4737 | + a: object | | 4738 | + * | | 4739 | + b: object | | 4740 | + """) | | 4741 | + generated = self.clinic.parse(block) | | 4742 | + self.assertIn('"O$O:func"', generated) | | 4743 | + | 4667 | 4744 | def test_limited_capi_double(self): | 4668 | 4745 | block = self.wrap_clinic_input(""" | 4669 | 4746 | func | @@ -4720,6 +4797,18 @@ def test_my_double_sum(self): | 4720 | 4797 | with self.assertRaises(TypeError): | 4721 | 4798 | func(1., "2") | 4722 | 4799 |
| | 4800 | + def test_required_kwonly(self): | | 4801 | + # test a required keyword-only parameter after an optional one | | 4802 | + func = _testclinic_limited.required_kwonly | | 4803 | + self.assertEqual(func(1, 2, c=3, d=4), (1, 2, 3, 4)) | | 4804 | + self.assertEqual(func(1, c=3), (1, None, 3, None)) | | 4805 | + with self.assertRaisesRegex(TypeError, "argument 'c'"): | | 4806 | + func(1, 2) | | 4807 | + with self.assertRaisesRegex(TypeError, "argument 'c'"): | | 4808 | + func(1, 2, d=4) | | 4809 | + with self.assertRaises(TypeError): | | 4810 | + func(1, 2, 3) | | 4811 | + | 4723 | 4812 | def test_get_file_descriptor(self): | 4724 | 4813 | # test 'file descriptor' converter: call PyObject_AsFileDescriptor() | 4725 | 4814 | get_fd = _testclinic_limited.get_file_descriptor | |
Collapse file Copy file name to clipboard +3Lines changed: 3 additions & 0 deletions - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
| | 1 | +:c:func:`PyArg_ParseTupleAndKeywords` now supports required keyword-only arguments mixed with optional arguments. | | 2 | +``|`` can now be specified after ``$`` to end the required keyword-only arguments. | | 3 | +For example, the format string ``"O|O$O|O"`` corresponds to the Python signature ``f(a, b=None, *, c, d=None)``. |
Collapse file Copy file name to clipboardExpand all lines: Modules/_testcapi/getargs.c +40Lines changed: 40 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -765,6 +765,42 @@ gh_99240_clear_args(PyObject *self, PyObject *args) | 765 | 765 | Py_RETURN_NONE; | 766 | 766 | } | 767 | 767 |
| | 768 | +/* f(a, b=None, *, c, d=None) parsed with a cached _PyArg_Parser. */ | | 769 | +static PyObject * | | 770 | +getargs_fast_required_kwonly(PyObject *self, PyObject *args, PyObject *kwargs) | | 771 | +{ | | 772 | + static const char * const keywords[] = {"a", "b", "c", "d", NULL}; | | 773 | + static _PyArg_Parser parser = { | | 774 | + .format = "O|O$O|O:getargs_fast_required_kwonly", | | 775 | + .keywords = keywords, | | 776 | + }; | | 777 | + PyObject *a, *b = Py_None, *c, *d = Py_None; | | 778 | + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &parser, | | 779 | + &a, &b, &c, &d)) | | 780 | + { | | 781 | + return NULL; | | 782 | + } | | 783 | + return Py_BuildValue("OOOO", a, b, c, d); | | 784 | +} | | 785 | + | | 786 | +/* f(a, b, *, c, d) parsed with a cached _PyArg_Parser. */ | | 787 | +static PyObject * | | 788 | +getargs_fast_kwonly(PyObject *self, PyObject *args, PyObject *kwargs) | | 789 | +{ | | 790 | + static const char * const keywords[] = {"a", "b", "c", "d", NULL}; | | 791 | + static _PyArg_Parser parser = { | | 792 | + .format = "OO$OO:getargs_fast_kwonly", | | 793 | + .keywords = keywords, | | 794 | + }; | | 795 | + PyObject *a, *b, *c, *d; | | 796 | + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &parser, | | 797 | + &a, &b, &c, &d)) | | 798 | + { | | 799 | + return NULL; | | 800 | + } | | 801 | + return Py_BuildValue("OOOO", a, b, c, d); | | 802 | +} | | 803 | + | 768 | 804 | static PyMethodDef test_methods[] = { | 769 | 805 | {"get_args", get_args, METH_VARARGS}, | 770 | 806 | {"get_kwargs", _PyCFunction_CAST(get_kwargs), METH_VARARGS|METH_KEYWORDS}, | @@ -809,6 +845,10 @@ static PyMethodDef test_methods[] = { | 809 | 845 | {"getargs_z_hash", getargs_z_hash, METH_VARARGS}, | 810 | 846 | {"getargs_z_star", getargs_z_star, METH_VARARGS}, | 811 | 847 | {"parse_tuple_and_keywords", parse_tuple_and_keywords, METH_VARARGS}, | | 848 | + {"getargs_fast_required_kwonly", | | 849 | + _PyCFunction_CAST(getargs_fast_required_kwonly), METH_VARARGS|METH_KEYWORDS}, | | 850 | + {"getargs_fast_kwonly", _PyCFunction_CAST(getargs_fast_kwonly), | | 851 | + METH_VARARGS|METH_KEYWORDS}, | 812 | 852 | {"gh_99240_clear_args", gh_99240_clear_args, METH_VARARGS}, | 813 | 853 | {"test_w_code_invalid", test_w_code_invalid, METH_NOARGS}, | 814 | 854 | {NULL}, | |
Collapse file Copy file name to clipboardExpand all lines: Modules/_testclinic_limited.c +22Lines changed: 22 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -129,9 +129,31 @@ static PyMethodDef tester_methods[] = { | 129 | 129 | MY_FLOAT_SUM_METHODDEF | 130 | 130 | MY_DOUBLE_SUM_METHODDEF | 131 | 131 | GET_FILE_DESCRIPTOR_METHODDEF | | 132 | + REQUIRED_KWONLY_METHODDEF | 132 | 133 | {NULL, NULL} | 133 | 134 | }; | 134 | 135 |
| | 136 | +/*[clinic input] | | 137 | +required_kwonly | | 138 | +
| | 139 | + a: object | | 140 | + b: object = None | | 141 | + * | | 142 | + c: object | | 143 | + d: object = None | | 144 | +
| | 145 | +Mix an optional parameter with a required keyword-only one. | | 146 | +[clinic start generated code]*/ | | 147 | + | | 148 | +static PyObject * | | 149 | +required_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, | | 150 | + PyObject *d) | | 151 | +/*[clinic end generated code: output=8e9a974afa614cd6 input=297ec6487f6373dd]*/ | | 152 | +{ | | 153 | + return Py_BuildValue("OOOO", a, b, c, d); | | 154 | +} | | 155 | + | | 156 | + | 135 | 157 | static struct PyModuleDef _testclinic_module = { | 136 | 158 | PyModuleDef_HEAD_INIT, | 137 | 159 | .m_name = "_testclinic_limited", | |
Collapse file Copy file name to clipboardExpand all lines: Modules/clinic/_testclinic_limited.c.h +33-1Lines changed: 33 additions & 1 deletion
|