Merge branch 'main' into babenek-patch-1

.github/workflows/check.yml

@@ -161,6 +161,8 @@ jobs:
         cd docs
         pip install -r requirements.txt
         make html
+        cd source
+        python -m sphinx -T -E -b html -d _build/doctrees -D language=en . ./_html
 
     # # # yapf
 

.readthedocs.yml

@@ -5,14 +5,19 @@
 # Required
 version: 2
 
+# Use build.os instead of build.image on your configuration file https://blog.readthedocs.com/use-build-os-config/
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.8"
+
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
+  builder: html
   configuration: docs/source/conf.py
+  fail_on_warning: true
 
-# Optionally set the version of Python and requirements required to build your docs
 python:
-  version: "3.8"
   install:
+    - requirements: requirements.txt
     - requirements: docs/requirements.txt
-    - method: pip
-      path: .

credsweeper/common/keyword_checklist.txt

@@ -904,6 +904,7 @@ unstable
 until
 update
 upload
+used
 username
 using
 usually

credsweeper/rules/config.yaml

@@ -1,3 +1,95 @@
+- name: SECRET_PAIR
+  severity: medium
+  type: pattern
+  values:
+    - (?P<variable>[`'\"]?(?i:token|secret|key|키|암호|암호화|토큰)[`'\"]?)((\s)*[=:](\s)*)(?P<quote>[`'\"(])?(?P<value>\S{4,})(?(quote)[)`'\"])
+  filter_type:
+    - ValueAllowlistCheck
+  min_line_len: 10
+  required_substrings:
+    - token
+    - secret
+    - key
+    - ":"
+    - "/"
+    - "="
+    - 키
+    - 암호
+    - 암호화
+    - 토큰
+  doc_only: true
+
+- name: PASSWD_PAIR
+  severity: medium
+  type: pattern
+  values:
+    - (?P<variable>[`'\"]?(?i:(?<!id[ :/])pa[as]swo?r?ds?|pwd?|p/w|비밀번호|비번|패스워드|암호)[`'\"]?)((\s)*[=:](\s)*)(?P<quote>[`'\"(])?(?P<value>\S{4,})(?(quote)[)`'\"])
+  filter_type:
+    - ValueAllowlistCheck
+  min_line_len: 10
+  required_substrings:
+    - pass
+    - sword
+    - ":"
+    - "/"
+    - "="
+    - 비밀번호
+    - 비번
+    - 패스워드
+    - 암호
+  doc_only: true
+
+- name: IP_ID_PASSWORD_TRIPLE
+  severity: medium
+  type: pattern
+  values:
+    - (^|(?P<variable>(?i:\bip[\s/]+id[\s/]+pw[\s/:]*))|(?P<url>://)|\s)(?P<ip>[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2})((?P<lpar>\s*\()?\s*|(?(variable)[\s,/]+|\s*(?(url)[,]|[,/])\s*))[\w.-]{3,}[\s,/]+(?P<value>(?(lpar)[^)\s/]{4,}|(?(url)[^\s/]{4,}|[^\s]{4,})))
+  filter_type:
+    - ValueAllowlistCheck
+  min_line_len: 10
+  required_substrings:
+    - "."
+  doc_only: true
+
+- name: ID_PAIR_PASSWD_PAIR
+  severity: medium
+  type: pattern
+  values:
+    - (?P<ddash>--)?(?P<variable>\w*(?i:pa[as]swords?|passwd?|pwd|\bp/w|\bpw|비밀번호|비번|패스워드|암호))\s*?(?(ddash)[ =]|[:=/>-]{1,2})\s*?(?P<quote>[`'\"]+)?(?P<value>\S{3,}?)(?(quote)(?P=quote)|\b)
+    - (?P<ddash>--)?(?P<variable>(?i:user\s*)?(?i:id|login|account|root|admin|user|name|wifi|role|host|default|계정|아이디))\s*?(?(ddash)[ =]|[ :=])\s*?(?P<value>\S+)
+  filter_type:
+    - ValueAllowlistCheck
+  min_line_len: 10
+  required_substrings:
+    - pass
+    - sword
+    - p/w
+    - pw
+    - 비밀번호
+    - 비번
+    - 패스워드
+    - 암호
+  doc_only: true
+
+- name: ID_PASSWD_PAIR
+  severity: medium
+  type: pattern
+  values:
+    - (?P<variable>[\w.-]*(?i:(?P<id>\bid\b)|id\b|user|name|계정|아이디)[\w.-]*(?(id)[ :(/]+|[:(/]+)(?i:pa[as]swo?r?ds?|pwd?|비밀번호|비번|패스워드|암호))\)?(\s*->\s*|[ =:)(/]+|\s+is\s+|\s+are\s+|\s*는\s*|\s*은\s*|\s*설정은\s*)\(?(?P<id_value>[\w.-]{2,31})[ :\(/\"',]+(?P<value>[^\s}\])\"']{4,31})
+  filter_type:
+    - ValueAllowlistCheck
+    - ValueDictionaryKeywordCheck
+  min_line_len: 10
+  required_substrings:
+    - pw
+    - pass
+    - sword
+    - 비밀번호
+    - 비번
+    - 패스워드
+    - 암호
+  doc_only: true
+
 - name: PII
   severity: info
   type: keyword

credsweeper/rules/rule.py

@@ -48,6 +48,7 @@ class Rule:
     REQUIRED_REGEX = "required_regex"
     VALIDATIONS = "validations"
     DOC_AVAILABLE = "doc_available"  # True - by default
+    DOC_ONLY = "doc_only"  # False - by default
 
     def __init__(self, config: Config, rule_dict: Dict) -> None:
         self.config = config
@@ -75,6 +76,7 @@ def __init__(self, config: Config, rule_dict: Dict) -> None:
         self.__required_regex = re.compile(required_regex) if required_regex else None
         self.__min_line_len = int(rule_dict.get(Rule.MIN_LINE_LEN, MAX_LINE_LENGTH))
         self.__doc_available: bool = rule_dict.get(Rule.DOC_AVAILABLE, True)
+        self.__doc_only: bool = rule_dict.get(Rule.DOC_ONLY, False)
 
     def _malformed_rule_error(self, rule_dict: Dict, field: str):
         raise ValueError(f"Malformed rule '{self.__rule_name}'."
@@ -241,3 +243,8 @@ def min_line_len(self) -> int:
     def doc_available(self) -> bool:
         """doc_available getter"""
         return self.__doc_available
+
+    @cached_property
+    def doc_only(self) -> bool:
+        """doc_only getter"""
+        return self.__doc_only

credsweeper/scanner/scanner.py

@@ -94,10 +94,13 @@ def _is_available(self, rule: Rule) -> bool:
             return False
         if self.config.doc:
             # apply only available for doc scanning rules
-            if rule.doc_available:
+            if rule.doc_available or rule.doc_only:
                 return True
         else:
-            return True
+            if rule.doc_only:
+                return False
+            else:
+                return True
         return False
 
     def yield_rule_scanner(

docs/requirements.txt

@@ -1,4 +1,5 @@
-m2r2
-sphinx_rtd_theme==1.2.2
-sphinx-autodoc-typehints
-autodocsumm
+m2r2==0.3.2
+# ^ the version is latest which has no conflicts
+sphinx_rtd_theme==1.3.0
+sphinx-autodoc-typehints==1.24.0
+autodocsumm==0.2.11

requirements.txt

@@ -1,6 +1,6 @@
 # Common requirements
 beautifulsoup4==4.12.2
-GitPython==3.1.37
+GitPython==3.1.40
 google-auth-oauthlib==1.1.0
 humanfriendly==10.0
 lxml==4.9.3

tests/__init__.py

@@ -1,17 +1,17 @@
 from pathlib import Path
 
 # total number of files in test samples
-SAMPLES_FILES_COUNT: int = 114
+SAMPLES_FILES_COUNT: int = 120
 
 # credentials count after scan
-SAMPLES_CRED_COUNT: int = 129
-SAMPLES_CRED_LINE_COUNT: int = 148
+SAMPLES_CRED_COUNT: int = 383
+SAMPLES_CRED_LINE_COUNT: int = 402
 
 # credentials count after post-processing
-SAMPLES_POST_CRED_COUNT: int = 122
+SAMPLES_POST_CRED_COUNT: int = 293
 
 # with option --doc
-SAMPLES_IN_DOC = 80
+SAMPLES_IN_DOC = 426
 
 # archived credentials that are not found without --depth
 SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 16