Optimized IsFileValid (#65)

Converted the token search algorithm to BoyerMoore and to run on a thread pool.
When using NVMe, the major cost was token lookup on the CPU.

src/AI_Fix_InvalidSceneFileProtection/AI_Fix_InvalidSceneFileProtection.csproj

@@ -48,6 +48,7 @@
       <HintPath>..\..\packages\IllusionLibs.BepInEx.5.4.22\lib\net35\BepInEx.dll</HintPath>
       <Private>False</Private>
     </Reference>
+    <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="UnityEngine, Version=0.0.0.0, Culture=neutral, processorArchitecture=MSIL">
       <HintPath>..\..\packages\IllusionLibs.AIGirl.UnityEngine.CoreModule.2018.2.21.4\lib\net46\UnityEngine.dll</HintPath>

src/Core_Fix_InvalidSceneFileProtection/InvalidSceneFileProtection.cs

@@ -6,6 +6,8 @@
 using System.IO;
 using System.Linq;
 using System.Reflection.Emit;
+using System.Diagnostics;
+using System.Threading;
 
 namespace IllusionFixes
 {
@@ -41,6 +43,65 @@ private static bool OnClickImportPrefix(List<string> ___listPath, int ___select)
             return IsFileValid(path);
         }
 
+        class Chunk
+        {
+            public static int Size = 1 << 20;
+            public byte[] bytes = new byte[Size];
+            public int readed;
+        }
+
+        class SearchStatus
+        {
+            public volatile bool found;
+        }
+
+        class PoolAllocator<T> where T : new()
+        {
+            private Queue<T> _queue = new Queue<T>();
+            private volatile int _remaingNewCount;
+            private int _maxPool;
+
+            public PoolAllocator( int maxPool )
+            {
+                _maxPool = maxPool;
+                _remaingNewCount = maxPool;
+            }
+
+            public T Acquire()
+            {
+                lock (_queue)
+                {
+                    while (_queue.Count <= 0 && _remaingNewCount <= 0)
+                        Monitor.Wait(_queue, 2000);
+
+                    if (_queue.Count > 0)
+                        return _queue.Dequeue();
+
+                    --_remaingNewCount;
+                }
+
+                return new T();
+            }
+
+            public void Release( T value )
+            {
+                lock (_queue)
+                {
+                    _queue.Enqueue(value);
+                    Monitor.Pulse(_queue);
+                }
+            }
+
+            public void WaitAllReleased()
+            {
+                lock(_queue)
+                {
+                    while (_queue.Count < _maxPool - _remaingNewCount)
+                        Monitor.Wait(_queue, 2000);
+                }
+            }
+        }
+
         private static bool IsFileValid(string path)
         {
             if (!File.Exists(path)) return false;
@@ -51,26 +112,53 @@ private static bool IsFileValid(string path)
                 {
                     PngFile.SkipPng(fs);
 
-                    var searchers = ValidStudioTokens.Select(token => new KMPSearch(token)).ToArray();
+                    var searchers = ValidStudioTokens.Select(token => new BoyerMoore(token)).ToArray();
                     int maxTokenSize = ValidStudioTokens.Max(token => token.Length);
+
+                    PoolAllocator<Chunk> allocator = new PoolAllocator<Chunk>(Math.Max(4, System.Environment.ProcessorCount));
+                    SearchStatus status = new SearchStatus();
 
-                    byte[] chunks = new byte[2 << 20];
+                    void _Search( object _chunk )
+                    {
+                        Chunk chunk = (Chunk)_chunk;
+
+                        try
+                        {
+                            for (int i = 0; i < searchers.Length; ++i)
+                                if (searchers[i].Contains(chunk.bytes, chunk.readed))
+                                {
+                                    status.found = true;
+                                    break;
+                                }
+                        }
+                        finally
+                        {
+                            allocator.Release(chunk);
+                        }
+                    }
 
-                    //Search for tokens while reading a certain size
-                    while (true)
+                    while (!status.found)
                     {
-                        int readed = fs.Read(chunks, 0, chunks.Length);
+                        var chunk = allocator.Acquire();
+                        chunk.readed = fs.Read(chunk.bytes, 0, chunk.bytes.Length);
 
-                        for (int i = 0; i < searchers.Length; ++i)
-                            if (searchers[i].Search(chunks, readed))
-                                return true;
+                        System.Threading.ThreadPool.QueueUserWorkItem(_Search, chunk);
 
-                        if (chunks.Length != readed)
+                        if (fs.Position >= fs.Length)
                             break;
 
                         //Slide a little because there may be data on the border.
                         fs.Position -= maxTokenSize - 1;
                     }
+
+                    if( !status.found )
+                    {
+                        //Waiting for search to finish
+                        allocator.WaitAllReleased();
+                    }
+
+                    if (status.found)
+                        return true;
                 }
 
                 LogInvalid();

src/Core_Fix_InvalidSceneFileProtection/Util.cs

@@ -3,85 +3,69 @@
 using System.Linq;
 
 namespace IllusionFixes
-{        
-    internal class KMPSearch
+{
+    internal class BoyerMoore
     {
-        private readonly byte[] pattern;
-        private int[] lps; // Longest Proper Prefix which is also Suffix
+        private readonly byte[] _needle;
+        private readonly int[] _badMatchTable;
 
-        public KMPSearch(byte[] pattern)
+        public BoyerMoore(byte[] needle)
         {
-            this.pattern = pattern;
-            this.lps = ComputeLPSArray();
+            _needle = needle ?? throw new ArgumentNullException(nameof(needle));
+            _badMatchTable = BuildBadMatchTable(_needle);
         }
 
-        public bool Search(byte[] text, int n)
+        public bool Contains(byte[] haystack, int haystackLen)
         {
-            int m = pattern.Length;
+            if (haystack == null)
+                throw new ArgumentNullException(nameof(haystack));
 
-            int i = 0; // index for text[]
-            int j = 0; // index for pattern[]
+            int needleLen = _needle.Length;
 
-            while (i < n)
+            if (needleLen == 0)
+                return true;
+
+            if (needleLen > haystackLen)
+                return false;
+
+            int skip = 0;
+
+            while (haystackLen - skip >= needleLen)
             {
-                if (pattern[j] == text[i])
+                int i = needleLen - 1;
+                while (i >= 0 && _needle[i] == haystack[skip + i])
                 {
-                    j++;
-                    i++;
+                    i--;
                 }
-
-                if (j == m)
+                if (i < 0)
                 {
-                    return true; // Pattern found
+                    return true; // needle found in haystack
                 }
-                else if (i < n && pattern[j] != text[i])
+                else
                 {
-                    if (j != 0)
-                    {
-                        j = lps[j - 1];
-                    }
-                    else
-                    {
-                        i++;
-                    }
+                    skip += _badMatchTable[haystack[skip + i]];
                 }
             }
 
-            return false; // Pattern not found
+            return false; // needle not found in haystack
         }
 
-        private int[] ComputeLPSArray()
+        private int[] BuildBadMatchTable(byte[] needle)
         {
-            int m = pattern.Length;
-            int[] lps = new int[m];
-            int len = 0; // length of the previous longest prefix suffix
+            int len = needle.Length;
+            int[] table = new int[256];
 
-            lps[0] = 0;
-            int i = 1;
+            for (int i = 0; i < table.Length; i++)
+            {
+                table[i] = len;
+            }
 
-            while (i < m)
+            for (int i = 0; i < len - 1; i++)
             {
-                if (pattern[i] == pattern[len])
-                {
-                    len++;
-                    lps[i] = len;
-                    i++;
-                }
-                else
-                {
-                    if (len != 0)
-                    {
-                        len = lps[len - 1];
-                    }
-                    else
-                    {
-                        lps[i] = 0;
-                        i++;
-                    }
-                }
+                table[needle[i]] = len - i - 1;
             }
 
-            return lps;
+            return table;
         }
     }
 }

src/HS2_Fix_InvalidSceneFileProtection/HS2_Fix_InvalidSceneFileProtection.csproj

@@ -52,6 +52,7 @@
       <HintPath>..\..\packages\IllusionLibs.HoneySelect2.IL.2020.5.29.4\lib\net46\IL.dll</HintPath>
       <Private>False</Private>
     </Reference>
+    <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="UnityEngine, Version=0.0.0.0, Culture=neutral, processorArchitecture=MSIL">
       <HintPath>..\..\packages\IllusionLibs.HoneySelect2.UnityEngine.CoreModule.2018.4.11.4\lib\net46\UnityEngine.dll</HintPath>

src/KK_Fix_InvalidSceneFileProtection/KK-KKS_Fix_InvalidSceneFileProtection.csproj

@@ -48,6 +48,7 @@
       <HintPath>..\..\packages\IllusionLibs.BepInEx.5.4.22\lib\net35\BepInEx.dll</HintPath>
       <Private>False</Private>
     </Reference>
+    <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="UnityEngine, Version=0.0.0.0, Culture=neutral, processorArchitecture=MSIL">
       <HintPath>..\..\packages\IllusionLibs.Koikatu.UnityEngine.5.6.2.4\lib\net35\UnityEngine.dll</HintPath>