speed up file search and fix long path error

winPEAS/winPEASexe/Tests/SearchHelperTests.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.IO;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using winPEAS.Helpers;
+using winPEAS.Helpers.Search;
+using Path = Alphaleonis.Win32.Filesystem.Path;
+
+
+namespace winPEAS.Tests
+{
+    [TestClass]
+    public class SearchHelperTests
+    {
+        [TestMethod]
+        public void TestGetFilesFastBypassesMAX_PATHLimit()
+        {
+            // Create a folder with files that have names longer than 260 characters
+            string folder = "C:\\Temp\\TestFolder";
+            var createdirectory = new DirectoryInfo(folder);
+            createdirectory.Create();
+            for (int i = 0; i < 10; i++)
+            {
+                string longName = new string('a', 300);
+                string fileName = Path.Combine(folder, $"{longName}_{i}.txt");
+
+                // Use the fsutil command to create a file with a long name
+                ProcessStartInfo startInfo = new ProcessStartInfo("fsutil", $"file createnew {fileName} 0")
+                {
+                    UseShellExecute = true,
+                    CreateNoWindow = false,
+                    RedirectStandardOutput = true,
+                    RedirectStandardError = true,
+                    ErrorDialog = true,
+                    WindowStyle = ProcessWindowStyle.Normal
+
+                };
+                Process.Start(startInfo);
+
+            }
+
+            // Call the GetFilesFast method to get a list of all the files in the folder
+            List<CustomFileInfo> files = SearchHelper.GetFilesFast(folder);
+
+            // Get a list of all the files in the folder using System.IO.Directory.GetFiles
+            string[] directoryFiles = System.IO.Directory.GetFiles(folder);
+            List<FileInfo> expectedFiles = directoryFiles.Select(f => new FileInfo(f)).ToList();
+
+            // Make sure the lists have the same number of elements
+            Assert.AreEqual(expectedFiles.Count, files.Count);
+
+
+        }
+    }
+}

winPEAS/winPEASexe/winPEAS/Checks/FileAnalysis.cs

@@ -318,6 +318,74 @@ namespace winPEAS.Checks
                     Console.WriteLine(string.Format("Key = {0}, Value = {1}", kvp.Key, kvp.Value));
                 }*/
 
+                //double pb = 0;
+                //using (var progress = new ProgressBar())
+                //{
+                //    CheckRunner.Run(() =>
+                //    {
+                //        int num_threads = 8;
+                //        try
+                //        {
+                //            num_threads = Environment.ProcessorCount;
+                //        }
+                //        catch (Exception ex) { }
+
+                //        Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = num_threads }, f =>
+                //        {
+
+                //            foreach (var regex_obj in config.regular_expresions)
+                //            {
+                //                foreach (var regex in regex_obj.regexes)
+                //                {
+                //                    if (regex.disable != null && regex.disable.ToLower().Contains("winpeas"))
+                //                    {
+                //                        continue;
+                //                    }
+
+                //                    List<string> results = new List<string> { };
+
+                //                    var timer = new Stopwatch();
+                //                    if (Checks.IsDebug)
+                //                    {
+                //                        timer.Start();
+                //                    }
+
+
+                //                    try
+                //                    {
+                //                        string text = File.ReadAllText(f.FullPath);
+
+                //                        results = SearchContent(text, regex.regex, (bool)regex.caseinsensitive);
+                //                        if (results.Count > 0)
+                //                        {
+                //                            if (!foundRegexes.ContainsKey(regex_obj.name)) foundRegexes[regex_obj.name] = new Dictionary<string, Dictionary<string, List<string>>> { };
+                //                            if (!foundRegexes[regex_obj.name].ContainsKey(regex.name)) foundRegexes[regex_obj.name][regex.name] = new Dictionary<string, List<string>> { };
+
+                //                            foundRegexes[regex_obj.name][regex.name][f.FullPath] = results;
+                //                        }
+                //                    }
+                //                    catch (System.IO.IOException)
+                //                    {
+                //                        // Cannot read the file
+                //                    }
+
+                //                    if (Checks.IsDebug)
+                //                    {
+                //                        timer.Stop();
+
+                //                        TimeSpan timeTaken = timer.Elapsed;
+                //                        if (timeTaken.TotalMilliseconds > 20000)
+                //                            Beaprint.PrintDebugLine($"\nThe regex {regex.regex} took {timeTaken.TotalMilliseconds}s in {f.FullPath}");
+                //                    }
+                //                }
+                //            }
+                //            pb += (double)100 / files.Count;
+                //            progress.Report(pb / 100); //Value must be in [0..1] range
+                //        });
+                //    }, Checks.IsDebug);
+                //}
+
+
                 double pb = 0;
                 using (var progress = new ProgressBar())
                 {
@@ -332,7 +400,6 @@ namespace winPEAS.Checks
 
                         Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = num_threads }, f =>
                         {
-
                             foreach (var regex_obj in config.regular_expresions)
                             {
                                 foreach (var regex in regex_obj.regexes)
@@ -342,7 +409,7 @@ namespace winPEAS.Checks
                                         continue;
                                     }
 
-                                    List<string> results = new List<string> { };
+                                    Dictionary<string, List<string>> fileResults = new Dictionary<string, List<string>>();
 
                                     var timer = new Stopwatch();
                                     if (Checks.IsDebug)
@@ -350,18 +417,31 @@ namespace winPEAS.Checks
                                         timer.Start();
                                     }
 
-
                                     try
                                     {
-                                        string text = File.ReadAllText(f.FullPath);
+                                        using (StreamReader sr = new StreamReader(f.FullPath))
-
+                                        {
-                                        results = SearchContent(text, regex.regex, (bool)regex.caseinsensitive);
+                                            string line;
+                                            while ((line = sr.ReadLine()) != null)
+                                            {
+                                                List<string> results = SearchContent(line, regex.regex, (bool)regex.caseinsensitive);
                                                 if (results.Count > 0)
+                                                {
+                                                    if (!fileResults.ContainsKey(f.FullPath))
+                                                    {
+                                                        fileResults[f.FullPath] = new List<string>();
+                                                    }
+                                                    fileResults[f.FullPath].AddRange(results);
+                                                }
+                                            }
+                                        }
+
+                                        if (fileResults.Count > 0)
                                         {
                                             if (!foundRegexes.ContainsKey(regex_obj.name)) foundRegexes[regex_obj.name] = new Dictionary<string, Dictionary<string, List<string>>> { };
                                             if (!foundRegexes[regex_obj.name].ContainsKey(regex.name)) foundRegexes[regex_obj.name][regex.name] = new Dictionary<string, List<string>> { };
 
-                                            foundRegexes[regex_obj.name][regex.name][f.FullPath] = results;
+                                            foundRegexes[regex_obj.name][regex.name] = fileResults;
                                         }
                                     }
                                     catch (System.IO.IOException)
@@ -385,6 +465,7 @@ namespace winPEAS.Checks
                     }, Checks.IsDebug);
                 }
 
+
                 // Print results
                 foreach (KeyValuePair<string, Dictionary<string, Dictionary<string, List<string>>>> item in foundRegexes)
                 {

winPEAS/winPEASexe/winPEAS/Helpers/Search/SearchHelper.cs

@@ -5,6 +5,8 @@ using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
+using DirectoryInfo = Alphaleonis.Win32.Filesystem.DirectoryInfo;
 
 namespace winPEAS.Helpers.Search
 {
@@ -39,12 +41,131 @@ namespace winPEAS.Helpers.Search
             ".png", ".psd", ".raw", ".svg", ".svgz", ".tif", ".tiff", ".webp",
         };
 
+        //public static List<CustomFileInfo> GetFilesFast(string folder, string pattern = "*", HashSet<string> excludedDirs = null, bool isFoldersIncluded = false)
+        //{
+        //    ConcurrentBag<CustomFileInfo> files = new ConcurrentBag<CustomFileInfo>();
+        //    IEnumerable<DirectoryInfo> startDirs = GetStartDirectories(folder, files, pattern, isFoldersIncluded);
+        //    IList<DirectoryInfo> startDirsExcluded = new List<DirectoryInfo>();
+        //    IList<string> known_dirs = new List<string>();
+
+        //    if (excludedDirs != null)
+        //    {
+        //        foreach (var startDir in startDirs)
+        //        {
+        //            bool shouldAdd = true;
+        //            string startDirLower = startDir.FullName.ToLower();
+
+        //            shouldAdd = !excludedDirs.Contains(startDirLower);
+
+        //            if (shouldAdd)
+        //            {
+        //                startDirsExcluded.Add(startDir);
+        //            }
+        //        }
+        //    }
+        //    else
+        //    {
+        //        startDirsExcluded = startDirs.ToList();
+        //    }
+
+        //    Parallel.ForEach(startDirsExcluded, (d) =>
+        //    {
+        //        Parallel.ForEach(GetStartDirectories(d.FullName, files, pattern, isFoldersIncluded), (dir) =>
+        //        {
+        //            GetFiles(dir.FullName, pattern).ForEach(
+        //                (f) =>
+        //                {
+        //                    if (!StaticExtensions.Contains(f.Extension.ToLower()))
+        //                    {
+        //                        // It should always be lesss than 260, but some times it isn't so this will bypass that file
+        //                        //if (Checks.Checks.IsLongPath || f.FullName.Length <= 260)
+        //                        //{
+        //                        CustomFileInfo file_info = new CustomFileInfo(f.Name, f.Extension, f.FullName, f.Length, false);
+        //                        files.Add(file_info);
+
+        //                        CustomFileInfo file_dir = new CustomFileInfo(f.Directory.Name, "", f.Directory.FullName, 0, true);
+        //                        if (!known_dirs.Contains(file_dir.FullPath))
+        //                        {
+        //                            known_dirs.Add(file_dir.FullPath);
+        //                            files.Add(file_dir);
+        //                        }
+        //                        //}
+        //                        //else if (f.FullName.Length > 260)
+        //                        //Beaprint.LongPathWarning(f.FullName);
+        //                    }
+        //                }
+        //                );
+        //        });
+        //    });
+
+        //    return files.ToList();
+        //}
+
+        //private static List<FileInfo> GetFiles(string folder, string pattern = "*")
+        //{
+        //    DirectoryInfo dirInfo;
+        //    DirectoryInfo[] directories;
+        //    try
+        //    {
+        //        dirInfo = new DirectoryInfo(folder);
+        //        directories = dirInfo.GetDirectories();
+
+        //        if (directories.Length == 0)
+        //        {
+        //            return new List<FileInfo>(dirInfo.GetFiles(pattern));
+        //        }
+        //    }
+        //    catch (UnauthorizedAccessException)
+        //    {
+        //        return new List<FileInfo>();
+        //    }
+        //    catch (PathTooLongException)
+        //    {
+        //        return new List<FileInfo>();
+        //    }
+        //    catch (DirectoryNotFoundException)
+        //    {
+        //        return new List<FileInfo>();
+        //    }
+        //    catch (Exception)
+        //    {
+        //        return new List<FileInfo>();
+        //    }
+
+        //    List<FileInfo> result = new List<FileInfo>();
+
+        //    foreach (var d in directories)
+        //    {
+        //        result.AddRange(GetFiles(d.FullName, pattern));
+        //    }
+
+        //    try
+        //    {
+        //        result.AddRange(dirInfo.GetFiles(pattern));
+        //    }
+        //    catch (UnauthorizedAccessException)
+        //    {
+        //    }
+        //    catch (PathTooLongException)
+        //    {
+        //    }
+        //    catch (DirectoryNotFoundException)
+        //    {
+        //    }
+        //    catch (Exception)
+        //    {
+        //    }
+
+        //    return result;
+        //}
+
+
         public static List<CustomFileInfo> GetFilesFast(string folder, string pattern = "*", HashSet<string> excludedDirs = null, bool isFoldersIncluded = false)
         {
             ConcurrentBag<CustomFileInfo> files = new ConcurrentBag<CustomFileInfo>();
             IEnumerable<DirectoryInfo> startDirs = GetStartDirectories(folder, files, pattern, isFoldersIncluded);
             IList<DirectoryInfo> startDirsExcluded = new List<DirectoryInfo>();
-            IList<string> known_dirs = new List<string>();
+            ConcurrentDictionary<string, byte> known_dirs = new ConcurrentDictionary<string, byte>();
 
             if (excludedDirs != null)
             {
@@ -68,37 +189,27 @@ namespace winPEAS.Helpers.Search
 
             Parallel.ForEach(startDirsExcluded, (d) =>
             {
-                Parallel.ForEach(GetStartDirectories(d.FullName, files, pattern, isFoldersIncluded), (dir) =>
+                var foundFiles = GetFiles(d.FullName, pattern);
+                foreach (var f in foundFiles)
                 {
-                    GetFiles(dir.FullName, pattern).ForEach(
+                    if (f != null && !StaticExtensions.Contains(f.Extension.ToLower()))
-                        (f) =>
-                        {
-                            if (!StaticExtensions.Contains(f.Extension.ToLower()))
-                            {
-                                // It should always be lesss than 260, but some times it isn't so this will bypass that file
-                                if (Checks.Checks.IsLongPath || f.FullName.Length <= 260)
                     {
                         CustomFileInfo file_info = new CustomFileInfo(f.Name, f.Extension, f.FullName, f.Length, false);
                         files.Add(file_info);
 
                         CustomFileInfo file_dir = new CustomFileInfo(f.Directory.Name, "", f.Directory.FullName, 0, true);
-                                    if (!known_dirs.Contains(file_dir.FullPath))
+                        if (known_dirs.TryAdd(file_dir.FullPath, 0))
                         {
-                                        known_dirs.Add(file_dir.FullPath);
                             files.Add(file_dir);
                         }
                     }
-                                else if (f.FullName.Length > 260)
-                                    Beaprint.LongPathWarning(f.FullName);
                 }
-                        }
-                        );
-                });
             });
 
             return files.ToList();
         }
 
+
         private static List<FileInfo> GetFiles(string folder, string pattern = "*")
         {
             DirectoryInfo dirInfo;
@@ -130,16 +241,22 @@ namespace winPEAS.Helpers.Search
                 return new List<FileInfo>();
             }
 
-            List<FileInfo> result = new List<FileInfo>();
+            ConcurrentBag<FileInfo> result = new ConcurrentBag<FileInfo>();
 
-            foreach (var d in directories)
+            Parallel.ForEach(directories, (d) =>
             {
-                result.AddRange(GetFiles(d.FullName, pattern));
+                foreach (var file in GetFiles(d.FullName, pattern))
+                {
+                    result.Add(file);
                 }
+            });
 
             try
             {
-                result.AddRange(dirInfo.GetFiles(pattern));
+                foreach (var file in dirInfo.GetFiles(pattern))
+                {
+                    result.Add(file);
+                }
             }
             catch (UnauthorizedAccessException)
             {
@@ -154,7 +271,7 @@ namespace winPEAS.Helpers.Search
             {
             }
 
-            return result;
+            return result.ToList();
         }
 
         private static IEnumerable<DirectoryInfo> GetStartDirectories(string folder, ConcurrentBag<CustomFileInfo> files, string pattern, bool isFoldersIncluded = false)

winPEAS/winPEASexe/winPEAS/packages.config

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="AlphaFS" version="2.2.6" targetFramework="net452" />
+</packages>

winPEAS/winPEASexe/winPEAS/winPEAS.csproj

@@ -114,12 +114,16 @@
     </StartupObject>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="AlphaFS, Version=2.2.0.0, Culture=neutral, PublicKeyToken=4d31a58f7d7ad5c9, processorArchitecture=MSIL">
+      <HintPath>..\packages\AlphaFS.2.2.6\lib\net452\AlphaFS.dll</HintPath>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="System.DirectoryServices.AccountManagement" />
     <Reference Include="System.Management" />
     <Reference Include="System.Security" />
     <Reference Include="System.ServiceProcess" />
+    <Reference Include="System.Transactions" />
     <Reference Include="System.Web.Extensions" />
     <Reference Include="System.Windows.Forms" />
     <Reference Include="System.Xml.Linq" />
@@ -696,6 +700,7 @@
       <Link>sensitive_files.yaml</Link>
     </EmbeddedResource>
     <None Include="App.config" />
+    <None Include="packages.config" />
     <None Include="TaskScheduler\V1\TaskSchedulerV1Schema.xsd">
       <SubType>Designer</SubType>
     </None>