Merge pull request #51 from jepperaskdk/pr/optional-keyword

Added support for optional argument for all 3 parsers.

.github/workflows/python-package.yml

@@ -38,13 +38,11 @@ jobs:
     - name: Test typing with mypy
       run: |
         mypy --config-file mypy.ini
-    # - name: Test docstrings with pydoctest
-    #   run: |
-    #     pydoctest
     - name: Test with pytest and coverage
       run: |
         coverage run -m pytest
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v1
-      with:
-        fail_ci_if_error: true
+    # - name: "Upload coverage to Codecov"
+    #   uses: codecov/codecov-action@v4
+    #   with:
+    #     fail_ci_if_error: true
+    #     token: ${{ secrets.CODECOV_TOKEN }} # required

changelog.md

@@ -0,0 +1,11 @@
+# Changelog
+
+## [0.2.0] - 2024-08-09
+
+### Added
+
+- [Breaking] Support for "optional" in all Google, Numpy and Sphinx parsers. This is breaking since it will start requiring optional parameters to be marked as optional in docstrings.
+
+## [<=0.1.22]
+
+- Versions below 0.2.0 were not tracked by this document. See [releases](https://github.com/jepperaskdk/pydoctest/releases) on GitHub.

pydoctest/main.py

@@ -46,7 +46,7 @@ def validate(self, modules: Optional[List[str]] = None) -> ValidationResult:
         """Validate the found modules using the provided reporter.
 
         Args:
-            modules (Optional[List[str]]): Optionally, specify directly the modules rather than discover.
+            modules (Optional[List[str]], optional): Optionally, specify directly the modules rather than discover.
 
         Returns:
             ValidationResult: Information about whether validation succeeded.
@@ -183,7 +183,7 @@ def get_configuration(root_dir: str, config_path: Optional[str] = None) -> Confi
 
     Args:
         root_dir (str): The directory to search in.
-        config_path (Optional[str]): [description]. Defaults to None.
+        config_path (Optional[str], optional): [description]. Defaults to None.
 
     Returns:
         Configuration: Either a configuration matching the specified/found one, or a default one.
@@ -207,7 +207,7 @@ def get_reporter(config: Configuration, reporter: Optional[str] = None) -> Repor
 
     Args:
         config (Configuration): The configuration currently used.
-        reporter (Optional[str]): Desired reporter [text | json]
+        reporter (Optional[str], optional): Desired reporter [text | json]
 
     Raises:
         Exception: Raised if desired reporter does not exist.

pydoctest/parsers/google_parser.py

@@ -17,10 +17,7 @@
 # Regex matches [name] ([type]): [description] with some extra whitespace
 # e.g. this would also match: a      (  int   )   :    kmdkfmdf
 # It terminates with .* meaning a new match is the terminator. This should support multiline descriptions without having to consider tabs/indentation.
-if sys.version_info[:2] >= (3,10):
-    ARGUMENT_REGEX = re.compile(r"\s*(?P<name>(\w+))\s*\((?P<type>[\w\.\[\], \'\|]+)\)\s*:(.*)")
-else:
-    ARGUMENT_REGEX = re.compile(r"\s*(?P<name>(\w+))\s*\((?P<type>[\w\.\[\], \']+)\)\s*:(.*)")
+ARGUMENT_REGEX = re.compile(r"\s*(?P<name>(\w+))\s*\((?P<type>[\w\.\[\], \'\|^\w]+?)(?P<optional>, optional)?\)\s*:(.*)")
 
 
 class GoogleParser(Parser):
@@ -145,10 +142,11 @@ def get_parameters(self, doc: str, module_type: ModuleType) -> List[Parameter]:
             start, end = match.span()
             name = match.group('name').strip()
             type = match.group('type').strip()
+            optional = match.group('optional')
 
             try:
                 located_type = get_type_from_module(type, module_type)
-                parameters.append(Parameter(name, located_type.type))
+                parameters.append(Parameter(name, located_type.type, optional is not None))
             except UnknownTypeException:
                 raise ParseException(f"Unknown type '{type}' in '{sections[Section.ARGUMENTS][start:end].strip()}'")
             except Exception:

pydoctest/parsers/numpy_parser.py

@@ -1,4 +1,5 @@
 import re
+import os
 import sys
 
 from types import ModuleType
@@ -28,12 +29,8 @@ def __init__(self) -> None:
         ]
         # Split by word, newline, a number of '-' and newline
         self.section_regex = re.compile("([a-zA-Z]+)\n[-]+\n")
-        if sys.version_info[:2] >= (3,10):
-            self.parameter_regex = re.compile(r"(\w+)\s*:\s*([\w\[\], \|]+)")
-            self.returns_with_name_regex = re.compile(r"(\w+)\s*:\s*([\w\[\], \|]+)")
-        else:
-            self.parameter_regex = re.compile(r"(\w+)\s*:\s*([\w\[\], ]+)")
-            self.returns_with_name_regex = re.compile(r"(\w+)\s*:\s*([\w\[\], ]+)")
+        self.parameter_regex = re.compile(r"(\w+)\s*:\s*([\w\[\], \| \^\w]+?)(?:(, optional)|$)")
+        self.returns_with_name_regex = re.compile(r"(\w+)\s*:\s*([\w\[\], \|]+)")
 
     def get_exceptions_raised(self, doc: str) -> List[str]:
         """Returns the exceptions listed as raised in the docstring.
@@ -115,16 +112,19 @@ def get_parameters(self, doc: str, module_type: ModuleType) -> List[Parameter]:
                 raise ParseException()
 
             parameters_section = splits[parameters_idx + 1]
-            matches = self.parameter_regex.findall(parameters_section)
+            parameters: List[Parameter] = []
+
+            for parameters_line in parameters_section.split('\n'):
+                matches = self.parameter_regex.findall(parameters_line)
+
+                for param_name, param_type, optional in matches:
+                    located_type = get_type_from_module(param_type, module_type)
+                    parameters.append(Parameter(param_name, located_type.type, len(optional.strip()) > 0))
 
             # If we have a parameters section with text, but no matches, we must have bad formatting
-            if len(parameters_section) > 0 and len(matches) == 0:
+            if len(parameters_section) > 0 and len(parameters) == 0:
                 raise ParseException()
-
-            parameters: List[Parameter] = []
-            for param_name, param_type in matches:
-                located_type = get_type_from_module(param_type, module_type)
-                parameters.append(Parameter(param_name, located_type.type))
+
             return parameters
         except Exception:
             raise ParseException()

pydoctest/parsers/parser.py

@@ -11,15 +11,17 @@ class Section(IntEnum):
 
 
 class Parameter():
-    def __init__(self, name: str, t: Type) -> None:
+    def __init__(self, name: str, t: Type, is_optional: bool) -> None:
         """Instantiates a function parameter.
 
         Args:
             name (str): The name of the argument.
             t (Type): The type of the argument
+            is_optional (bool): Whether the argument is optional.
         """
         self.name = name
         self.type = t
+        self.is_optional = is_optional
 
 
 class Parser():

pydoctest/parsers/sphinx_parser.py

@@ -21,12 +21,8 @@ def __init__(self) -> None:
         super().__init__()
         self.parameter_name_regex = re.compile(r":param\s+(\w+):")
 
-        if sys.version_info[:2] >= (3,10):
-            self.parameter_type_regex = re.compile(r":type\s+\w+:\s*([\w\[\], \|]+)")
-            self.return_type_regex = re.compile(r":rtype:\s*([\w\[\], \|]+)")
-        else:
-            self.parameter_type_regex = re.compile(r":type\s+\w+:\s*([\w\[\], ]+)")
-            self.return_type_regex = re.compile(r":rtype:\s*([\w\[\], ]+)")
+        self.parameter_type_regex = re.compile(r":type\s+\w+:\s*([\w\[\], \|\^\w]+?)(?:(, optional)|$)")
+        self.return_type_regex = re.compile(r":rtype:\s*([\w\[\], \|]+)")
 
         self.raises_regex = re.compile(r":raises\s+(\w+):")
 
@@ -105,8 +101,9 @@ def get_parameters(self, doc: str, module_type: ModuleType) -> List[Parameter]:
                     match = self.parameter_type_regex.match(line)
                     if match is not None:
                         var_type = match.groups()[0]
+                        var_optional = match.groups()[1]
                         located_type = get_type_from_module(var_type, module_type)
-                        parameters.append(Parameter(var_name, located_type.type))
+                        parameters.append(Parameter(var_name, located_type.type, var_optional is not None))
                         var_name = None
             if var_name is not None:
                 # There must have been an error parsing :type:

pydoctest/reporters/text_reporter.py

@@ -51,7 +51,7 @@ def get_function_output(self, result: FunctionValidationResult, class_name: Opti
 
         Args:
             result (FunctionValidationResult): The result from running Pydoctest on the function.
-            class_name (Optional[str]): Optionally which class this function is run within.
+            class_name (Optional[str], optional): Optionally which class this function is run within.
         Returns:
             str: The output from the function.
         """

pydoctest/utilities.py

@@ -173,7 +173,7 @@ def parse_cli_list(content: str, separator: str = ',') -> List[str]:
 
     Args:
         content (str): The string coming from a cli command.
-        separator (str): The separator to split the list by. Defaults to ','.
+        separator (str, optional): The separator to split the list by. Defaults to ','.
 
     Returns:
         List[str]: The list-items.

pydoctest/validation.py

@@ -223,7 +223,7 @@ def __get_docstring_range(fn: FunctionType, module_type: ModuleType, docstring:
     Args:
         fn (FunctionType): The function to validate.
         module_type (ModuleType): The module from which the function was extracted.
-        docstring (Optional[str]): Optionally, the docstring.
+        docstring (Optional[str], optional): Optionally, the docstring.
 
     Returns:
         Optional[Range]: The range, if found.
@@ -279,7 +279,7 @@ def validate_function(fn: FunctionType, config: Configuration, module_type: Modu
         return result
 
     sig = inspect.signature(fn)
-    sig_parameters = [Parameter(name, proxy.annotation) for name, proxy in sig.parameters.items() if name != "self"]
+    sig_parameters = [Parameter(name, proxy.annotation, proxy.default is not inspect.Parameter.empty) for name, proxy in sig.parameters.items() if name != "self"]
     sig_return_type = type(None) if sig.return_annotation is None else sig.return_annotation
 
     try:
@@ -319,6 +319,14 @@ def validate_function(fn: FunctionType, config: Configuration, module_type: Modu
             result.fail_reason = f"Argument type differ. Argument '{sigparam.name}' was expected (from signature) to have type '{sigparam.type}', but has (in docs) type '{docparam.type}'"
             result.range = __get_docstring_range(fn, module_type, doc)
             return result
+
+        if sigparam.is_optional != docparam.is_optional:
+            result.result = ResultType.FAILED
+            sig_optional = 'optional' if sigparam.is_optional else 'not optional'
+            doc_optional = 'optional' if docparam.is_optional else 'not optional'
+            result.fail_reason = f"Argument optional differs. Argument '{sigparam.name}' was expected (from signature) to be {sig_optional}, but is (in docs) {doc_optional}"
+            result.range = __get_docstring_range(fn, module_type, doc)
+            return result
 
     # Validate exceptions raised
     if config.fail_on_raises_section:

pydoctest/version.py

@@ -1 +1 @@
-VERSION = "0.1.22"
+VERSION = "0.2.0"

tests/test_class/correct_class.py

@@ -90,3 +90,14 @@ def func_has_union_arg(self, a: Union[int, str]) -> None:
                 a (int | str): [description]
             """
             pass
+
+    def func_with_optional(self, a: int = 0) -> int:
+        """[summary]
+
+        Args:
+            a (int, optional): [description]
+
+        Returns:
+            int: [description]
+        """
+        pass

tests/test_class/incorrect_class.py

@@ -107,3 +107,25 @@ def func_parse_exception(self, a: int) -> int:
             THISDOESNTPARSE
         """
         pass
+
+    def func_optional_mismatch(self, a: int = 0) -> int:
+        """[summary]
+
+        Args:
+            a (int): [description]        <-- a should be marked optional
+
+        Returns:
+            int: [description]
+        """
+        pass
+
+    def func_optional_mismatch2(self, a: int) -> int:
+        """[summary]
+
+        Args:
+            a (int, optional): [description]        <-- a should not be marked optional
+
+        Returns:
+            int: [description]
+        """
+        pass

tests/test_class/test_class.py

@@ -125,6 +125,18 @@ def test_incorrect_class_func_type_mismatch(self) -> None:
         assert result.result == ResultType.FAILED
         assert result.fail_reason == "Argument type differ. Argument 'a' was expected (from signature) to have type '<class 'int'>', but has (in docs) type '<class 'float'>'"
 
+    def test_incorrect_class_func_optional_mismatch1(self) -> None:
+        config = Configuration.get_default_configuration()
+        result = validate_function(tests.test_class.incorrect_class.IncorrectTestClass.func_optional_mismatch, config, tests.test_class.incorrect_class)
+        assert result.result == ResultType.FAILED
+        assert result.fail_reason == "Argument optional differs. Argument 'a' was expected (from signature) to be optional, but is (in docs) not optional"
+
+    def test_incorrect_class_func_optional_mismatch2(self) -> None:
+        config = Configuration.get_default_configuration()
+        result = validate_function(tests.test_class.incorrect_class.IncorrectTestClass.func_optional_mismatch2, config, tests.test_class.incorrect_class)
+        assert result.result == ResultType.FAILED
+        assert result.fail_reason == "Argument optional differs. Argument 'a' was expected (from signature) to be not optional, but is (in docs) optional"
+
     # Test counts_class
     def test_counts_class(self) -> None:
         config = Configuration.get_configuration_from_path("tests/test_class/pydoctest_get_counts.json")

tests/test_parsers/google_class.py

@@ -30,3 +30,12 @@ def function_with_pipe(self, a: Union[int, float]) -> Union[int, float]:
         """
 
         pass
+
+    def func_optional_argument(self, a: int, b: int = 0) -> None:
+        """Function with optional argument
+
+        Args:
+            a (int): [description]
+            b (int, optional): [description]
+        """
+        pass

tests/test_parsers/numpy_class.py

@@ -137,6 +137,18 @@ def func_parse_exception(self, a: int) -> int:
         """
         pass
 
+    def func_missing_optional_argument(self, a: int, b: int = 0) -> None:
+        """Function with optional argument
+
+        Parameters
+        ----------
+        a : int
+            [description]
+        b : int
+            Missing ", optional" on this argument.
+        """
+        pass
+
 
 class CorrectTestClass():
 
@@ -275,6 +287,18 @@ def func_no_summary(self) -> None:
         """
         pass
 
+    def func_optional_argument(self, a: int, b: int = 0) -> None:
+        """Function with optional argument
+
+        Parameters
+        ----------
+        a : int
+            [description]
+        b : int, optional
+            [description]
+        """
+        pass
+
 
 class RaisesClass():
 

tests/test_parsers/sphinx_class.py

@@ -199,6 +199,16 @@ def func_no_summary(self) -> None:
         """
         pass
 
+    def func_optional_argument(self, a: int, b: int = 0) -> None:
+        """Function with optional argument
+
+        :param a: [description]
+        :type a: int
+        :param b: [description]
+        :type b: int, optional
+        """
+        pass
+
 
 class RaisesClass():
 

tests/test_parsers/test_google_parser.py

@@ -197,3 +197,16 @@ def test_function_with_pipe(self) -> None:
         assert len(params) == 1
         assert params[0].name == 'a'
         assert params[0].type == int | float
+
+    def test_func_with_optional_argument(self) -> None:
+        parser = GoogleParser()
+        doc = pydoc.getdoc(tests.test_parsers.google_class.GoogleClass.func_optional_argument)
+        parameters = parser.get_parameters(doc, tests.test_parsers.google_class)
+        assert len(parameters) == 2
+        assert parameters[0].name == 'a'
+        assert parameters[0].type == int
+        assert parameters[0].is_optional is False
+
+        assert parameters[1].name == 'b'
+        assert parameters[1].type == int
+        assert parameters[1].is_optional is True

tests/test_parsers/test_numpy_parser.py

@@ -180,3 +180,16 @@ def test_func_with_generics(self) -> None:
 
         return_type = parser.get_return_type(doc, tests.test_parsers.numpy_class)
         assert return_type == Dict[str, Any]
+
+    def test_func_with_optional_argument(self) -> None:
+        parser = NumpyParser()
+        doc = pydoc.getdoc(tests.test_parsers.numpy_class.CorrectTestClass.func_optional_argument)
+        parameters = parser.get_parameters(doc, tests.test_parsers.numpy_class)
+        assert len(parameters) == 2
+        assert parameters[0].name == 'a'
+        assert parameters[0].type == int
+        assert parameters[0].is_optional is False
+
+        assert parameters[1].name == 'b'
+        assert parameters[1].type == int
+        assert parameters[1].is_optional is True