Switch to chardet for stable behavior

charset_normalizer has stability issues

Reference: jawah/charset_normalizer#520
Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

requirements.txt

@@ -1,7 +1,7 @@
 attrs==22.2.0
 beautifulsoup4==4.11.2
 certifi==2022.12.7
-charset-normalizer==3.0.1
+chardet==5.2.0
 click==8.1.3
 idna==3.4
 pip==23.0

setup.cfg

@@ -41,9 +41,9 @@ python_requires = >=3.8
 
 install_requires =
     attrs >= 18.1, !=20.1.0
-    Beautifulsoup4 >= 4.0.0
+    Beautifulsoup4[chardet] >= 4.0.0
     click >= 6.7, !=7.0
-    requests >= 2.7.0
+    requests[use_chardet_on_py3] >= 2.7.0
     saneyaml >= 0.5.2
     text_unidecode >= 1.0
 

tests/test_paths.py

@@ -76,8 +76,11 @@ def test_safe_path_posix_style_french_char(self):
 
     def test_safe_path_posix_style_chinese_char(self):
         test = paths.safe_path(b'/includes/webform.compon\xd2\xaants.inc/')
-        expected = 'includes/webform.componNSnts.inc'
-        assert test == expected
+        expected = [
+            'includes/webform.componNSnts.inc',
+            'includes/webform.componS_nts.inc',
+        ]
+        assert test in expected
 
     def test_safe_path_windows_style_dots(self):
         test = paths.safe_path('\\includes\\..\\webform.components.inc\\')