diff --git a/winPEAS/winPEASexe/Tests/SearchHelperTests.cs b/winPEAS/winPEASexe/Tests/SearchHelperTests.cs new file mode 100644 index 0000000..ee602d9 --- /dev/null +++ b/winPEAS/winPEASexe/Tests/SearchHelperTests.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using winPEAS.Helpers; +using winPEAS.Helpers.Search; +using Path = Alphaleonis.Win32.Filesystem.Path; + + +namespace winPEAS.Tests +{ + [TestClass] + public class SearchHelperTests + { + [TestMethod] + public void TestGetFilesFastBypassesMAX_PATHLimit() + { + // Create a folder with files that have names longer than 260 characters + string folder = "C:\\Temp\\TestFolder"; + var createdirectory = new DirectoryInfo(folder); + createdirectory.Create(); + for (int i = 0; i < 10; i++) + { + string longName = new string('a', 300); + string fileName = Path.Combine(folder, $"{longName}_{i}.txt"); + + // Use the fsutil command to create a file with a long name + ProcessStartInfo startInfo = new ProcessStartInfo("fsutil", $"file createnew {fileName} 0") + { + UseShellExecute = true, + CreateNoWindow = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + ErrorDialog = true, + WindowStyle = ProcessWindowStyle.Normal + + }; + Process.Start(startInfo); + + } + + // Call the GetFilesFast method to get a list of all the files in the folder + List files = SearchHelper.GetFilesFast(folder); + + // Get a list of all the files in the folder using System.IO.Directory.GetFiles + string[] directoryFiles = System.IO.Directory.GetFiles(folder); + List expectedFiles = directoryFiles.Select(f => new FileInfo(f)).ToList(); + + // Make sure the lists have the same number of elements + Assert.AreEqual(expectedFiles.Count, files.Count); + + + } + } +} \ No newline at end of file diff --git a/winPEAS/winPEASexe/winPEAS/Checks/FileAnalysis.cs b/winPEAS/winPEASexe/winPEAS/Checks/FileAnalysis.cs index d6ff565..05335e0 100644 --- a/winPEAS/winPEASexe/winPEAS/Checks/FileAnalysis.cs +++ b/winPEAS/winPEASexe/winPEAS/Checks/FileAnalysis.cs @@ -318,6 +318,74 @@ namespace winPEAS.Checks Console.WriteLine(string.Format("Key = {0}, Value = {1}", kvp.Key, kvp.Value)); }*/ + //double pb = 0; + //using (var progress = new ProgressBar()) + //{ + // CheckRunner.Run(() => + // { + // int num_threads = 8; + // try + // { + // num_threads = Environment.ProcessorCount; + // } + // catch (Exception ex) { } + + // Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = num_threads }, f => + // { + + // foreach (var regex_obj in config.regular_expresions) + // { + // foreach (var regex in regex_obj.regexes) + // { + // if (regex.disable != null && regex.disable.ToLower().Contains("winpeas")) + // { + // continue; + // } + + // List results = new List { }; + + // var timer = new Stopwatch(); + // if (Checks.IsDebug) + // { + // timer.Start(); + // } + + + // try + // { + // string text = File.ReadAllText(f.FullPath); + + // results = SearchContent(text, regex.regex, (bool)regex.caseinsensitive); + // if (results.Count > 0) + // { + // if (!foundRegexes.ContainsKey(regex_obj.name)) foundRegexes[regex_obj.name] = new Dictionary>> { }; + // if (!foundRegexes[regex_obj.name].ContainsKey(regex.name)) foundRegexes[regex_obj.name][regex.name] = new Dictionary> { }; + + // foundRegexes[regex_obj.name][regex.name][f.FullPath] = results; + // } + // } + // catch (System.IO.IOException) + // { + // // Cannot read the file + // } + + // if (Checks.IsDebug) + // { + // timer.Stop(); + + // TimeSpan timeTaken = timer.Elapsed; + // if (timeTaken.TotalMilliseconds > 20000) + // Beaprint.PrintDebugLine($"\nThe regex {regex.regex} took {timeTaken.TotalMilliseconds}s in {f.FullPath}"); + // } + // } + // } + // pb += (double)100 / files.Count; + // progress.Report(pb / 100); //Value must be in [0..1] range + // }); + // }, Checks.IsDebug); + //} + + double pb = 0; using (var progress = new ProgressBar()) { @@ -332,7 +400,6 @@ namespace winPEAS.Checks Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = num_threads }, f => { - foreach (var regex_obj in config.regular_expresions) { foreach (var regex in regex_obj.regexes) @@ -342,7 +409,7 @@ namespace winPEAS.Checks continue; } - List results = new List { }; + Dictionary> fileResults = new Dictionary>(); var timer = new Stopwatch(); if (Checks.IsDebug) @@ -350,18 +417,31 @@ namespace winPEAS.Checks timer.Start(); } - try { - string text = File.ReadAllText(f.FullPath); + using (StreamReader sr = new StreamReader(f.FullPath)) + { + string line; + while ((line = sr.ReadLine()) != null) + { + List results = SearchContent(line, regex.regex, (bool)regex.caseinsensitive); + if (results.Count > 0) + { + if (!fileResults.ContainsKey(f.FullPath)) + { + fileResults[f.FullPath] = new List(); + } + fileResults[f.FullPath].AddRange(results); + } + } + } - results = SearchContent(text, regex.regex, (bool)regex.caseinsensitive); - if (results.Count > 0) + if (fileResults.Count > 0) { if (!foundRegexes.ContainsKey(regex_obj.name)) foundRegexes[regex_obj.name] = new Dictionary>> { }; if (!foundRegexes[regex_obj.name].ContainsKey(regex.name)) foundRegexes[regex_obj.name][regex.name] = new Dictionary> { }; - foundRegexes[regex_obj.name][regex.name][f.FullPath] = results; + foundRegexes[regex_obj.name][regex.name] = fileResults; } } catch (System.IO.IOException) @@ -385,6 +465,7 @@ namespace winPEAS.Checks }, Checks.IsDebug); } + // Print results foreach (KeyValuePair>>> item in foundRegexes) { diff --git a/winPEAS/winPEASexe/winPEAS/Helpers/Search/SearchHelper.cs b/winPEAS/winPEASexe/winPEAS/Helpers/Search/SearchHelper.cs index 2cb81e1..c5220f1 100644 --- a/winPEAS/winPEASexe/winPEAS/Helpers/Search/SearchHelper.cs +++ b/winPEAS/winPEASexe/winPEAS/Helpers/Search/SearchHelper.cs @@ -5,6 +5,8 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; +using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; +using DirectoryInfo = Alphaleonis.Win32.Filesystem.DirectoryInfo; namespace winPEAS.Helpers.Search { @@ -37,14 +39,133 @@ namespace winPEAS.Helpers.Search ".bmp", ".emf", ".gif", ".pm", ".jif", ".jfi", ".jfif", ".jpe", ".jpeg", ".jpg", ".png", ".psd", ".raw", ".svg", ".svgz", ".tif", ".tiff", ".webp", - }; + }; + + //public static List GetFilesFast(string folder, string pattern = "*", HashSet excludedDirs = null, bool isFoldersIncluded = false) + //{ + // ConcurrentBag files = new ConcurrentBag(); + // IEnumerable startDirs = GetStartDirectories(folder, files, pattern, isFoldersIncluded); + // IList startDirsExcluded = new List(); + // IList known_dirs = new List(); + + // if (excludedDirs != null) + // { + // foreach (var startDir in startDirs) + // { + // bool shouldAdd = true; + // string startDirLower = startDir.FullName.ToLower(); + + // shouldAdd = !excludedDirs.Contains(startDirLower); + + // if (shouldAdd) + // { + // startDirsExcluded.Add(startDir); + // } + // } + // } + // else + // { + // startDirsExcluded = startDirs.ToList(); + // } + + // Parallel.ForEach(startDirsExcluded, (d) => + // { + // Parallel.ForEach(GetStartDirectories(d.FullName, files, pattern, isFoldersIncluded), (dir) => + // { + // GetFiles(dir.FullName, pattern).ForEach( + // (f) => + // { + // if (!StaticExtensions.Contains(f.Extension.ToLower())) + // { + // // It should always be lesss than 260, but some times it isn't so this will bypass that file + // //if (Checks.Checks.IsLongPath || f.FullName.Length <= 260) + // //{ + // CustomFileInfo file_info = new CustomFileInfo(f.Name, f.Extension, f.FullName, f.Length, false); + // files.Add(file_info); + + // CustomFileInfo file_dir = new CustomFileInfo(f.Directory.Name, "", f.Directory.FullName, 0, true); + // if (!known_dirs.Contains(file_dir.FullPath)) + // { + // known_dirs.Add(file_dir.FullPath); + // files.Add(file_dir); + // } + // //} + // //else if (f.FullName.Length > 260) + // //Beaprint.LongPathWarning(f.FullName); + // } + // } + // ); + // }); + // }); + + // return files.ToList(); + //} + + //private static List GetFiles(string folder, string pattern = "*") + //{ + // DirectoryInfo dirInfo; + // DirectoryInfo[] directories; + // try + // { + // dirInfo = new DirectoryInfo(folder); + // directories = dirInfo.GetDirectories(); + + // if (directories.Length == 0) + // { + // return new List(dirInfo.GetFiles(pattern)); + // } + // } + // catch (UnauthorizedAccessException) + // { + // return new List(); + // } + // catch (PathTooLongException) + // { + // return new List(); + // } + // catch (DirectoryNotFoundException) + // { + // return new List(); + // } + // catch (Exception) + // { + // return new List(); + // } + + // List result = new List(); + + // foreach (var d in directories) + // { + // result.AddRange(GetFiles(d.FullName, pattern)); + // } + + // try + // { + // result.AddRange(dirInfo.GetFiles(pattern)); + // } + // catch (UnauthorizedAccessException) + // { + // } + // catch (PathTooLongException) + // { + // } + // catch (DirectoryNotFoundException) + // { + // } + // catch (Exception) + // { + // } + + // return result; + //} + public static List GetFilesFast(string folder, string pattern = "*", HashSet excludedDirs = null, bool isFoldersIncluded = false) { ConcurrentBag files = new ConcurrentBag(); IEnumerable startDirs = GetStartDirectories(folder, files, pattern, isFoldersIncluded); IList startDirsExcluded = new List(); - IList known_dirs = new List(); + ConcurrentDictionary known_dirs = new ConcurrentDictionary(); if (excludedDirs != null) { @@ -68,37 +189,27 @@ namespace winPEAS.Helpers.Search Parallel.ForEach(startDirsExcluded, (d) => { - Parallel.ForEach(GetStartDirectories(d.FullName, files, pattern, isFoldersIncluded), (dir) => + var foundFiles = GetFiles(d.FullName, pattern); + foreach (var f in foundFiles) { - GetFiles(dir.FullName, pattern).ForEach( - (f) => + if (f != null && !StaticExtensions.Contains(f.Extension.ToLower())) + { + CustomFileInfo file_info = new CustomFileInfo(f.Name, f.Extension, f.FullName, f.Length, false); + files.Add(file_info); + + CustomFileInfo file_dir = new CustomFileInfo(f.Directory.Name, "", f.Directory.FullName, 0, true); + if (known_dirs.TryAdd(file_dir.FullPath, 0)) { - if (!StaticExtensions.Contains(f.Extension.ToLower())) - { - // It should always be lesss than 260, but some times it isn't so this will bypass that file - if (Checks.Checks.IsLongPath || f.FullName.Length <= 260) - { - CustomFileInfo file_info = new CustomFileInfo(f.Name, f.Extension, f.FullName, f.Length, false); - files.Add(file_info); - - CustomFileInfo file_dir = new CustomFileInfo(f.Directory.Name, "", f.Directory.FullName, 0, true); - if (!known_dirs.Contains(file_dir.FullPath)) - { - known_dirs.Add(file_dir.FullPath); - files.Add(file_dir); - } - } - else if (f.FullName.Length > 260) - Beaprint.LongPathWarning(f.FullName); - } + files.Add(file_dir); } - ); - }); + } + } }); return files.ToList(); - } - + } + + private static List GetFiles(string folder, string pattern = "*") { DirectoryInfo dirInfo; @@ -130,16 +241,22 @@ namespace winPEAS.Helpers.Search return new List(); } - List result = new List(); + ConcurrentBag result = new ConcurrentBag(); - foreach (var d in directories) + Parallel.ForEach(directories, (d) => { - result.AddRange(GetFiles(d.FullName, pattern)); - } + foreach (var file in GetFiles(d.FullName, pattern)) + { + result.Add(file); + } + }); try { - result.AddRange(dirInfo.GetFiles(pattern)); + foreach (var file in dirInfo.GetFiles(pattern)) + { + result.Add(file); + } } catch (UnauthorizedAccessException) { @@ -154,7 +271,7 @@ namespace winPEAS.Helpers.Search { } - return result; + return result.ToList(); } private static IEnumerable GetStartDirectories(string folder, ConcurrentBag files, string pattern, bool isFoldersIncluded = false) diff --git a/winPEAS/winPEASexe/winPEAS/packages.config b/winPEAS/winPEASexe/winPEAS/packages.config new file mode 100644 index 0000000..168249c --- /dev/null +++ b/winPEAS/winPEASexe/winPEAS/packages.config @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/winPEAS/winPEASexe/winPEAS/winPEAS.csproj b/winPEAS/winPEASexe/winPEAS/winPEAS.csproj index eb90fe8..1a8efc9 100755 --- a/winPEAS/winPEASexe/winPEAS/winPEAS.csproj +++ b/winPEAS/winPEASexe/winPEAS/winPEAS.csproj @@ -114,12 +114,16 @@ + + ..\packages\AlphaFS.2.2.6\lib\net452\AlphaFS.dll + + @@ -696,6 +700,7 @@ sensitive_files.yaml + Designer