Fixed scoring for compression codecs

Author: dhondta

codext/__common__.py

@@ -1402,8 +1402,8 @@ def __score(prev_input, input, prev_encoding, encoding, codec, heuristic=False,
             except TypeError:
                 entr = entr(obj.entropy)
         if entr is not None:
-            # use a quadratic heuristic to compute a weight for the entropy delta, aligned on (100w,.1) and (200w,1)
-            d_entr = min(5.958194e-06 * obj.len**2 - .002381 * obj.len, 1) * abs(entr - entropy(new_input))
+            # use a quadratic heuristic to compute a weight for the entropy delta, aligned on (256,.2) and (512,1)
+            d_entr = min(3.04575e-06 * obj.len**2 + .000394 * obj.len, 1) * abs(entr - entropy(new_input))
             if d_entr <= .5:
                 s += .5 - d_entr
         # finally, if relevant, apply a custom bonus (e.g. when a regex pattern is matched)

codext/compressions/__init__.py

@@ -4,3 +4,9 @@
 from .lz78 import *
 from .pkzip import *
 
+
+for e in list_encodings("compression"):
+    ci = lookup(e, False)
+    ci.parameters['scoring']['entropy'] = 7.9
+    ci.parameters['scoring']['expansion_factor'] = lambda f: f
+

codext/compressions/gzipp.py

@@ -40,5 +40,5 @@ def gzip_decompress(data, errors="strict"):
     return r, len(r)
 
 
-add("gzip", gzip_compress, gzip_decompress, entropy=7.9)
+add("gzip", gzip_compress, gzip_decompress)
 

codext/compressions/lz77.py

@@ -70,5 +70,5 @@ def lz77_decompress(input, errors="strict"):
     return out, len(out)
 
 
-add("lz77", lz77_compress, lz77_decompress, entropy=7.9)
+add("lz77", lz77_compress, lz77_decompress)
 

codext/compressions/pkzip.py

@@ -46,11 +46,11 @@ def _decode(data, errors="strict"):
 
 
     add("pkzip_deflate", pkzip_encode(8), pkzip_decode(8, "deflate"), r"(?:(?:pk)?zip[-_])?deflate",
-        entropy=7.9, examples=__examples1__, guess=["deflate"])
+        examples=__examples1__, guess=["deflate"])
 
     add("pkzip_bzip2", pkzip_encode(12), pkzip_decode(12, "bzip2"), r"(?:(?:pk)?zip[-_])?bz(?:ip)?2",
-        entropy=7.9, examples=__examples2__, guess=["bz2"])
+        examples=__examples2__, guess=["bz2"])
 
     add("pkzip_lzma", pkzip_encode(14), pkzip_decode(14, "lzma"), r"(?:(?:pk)?zip[-_])?lzma",
-        entropy=7.9, examples=__examples3__, guess=["lzma"])
+        examples=__examples3__, guess=["lzma"])