diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Redis/RedisClientProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Redis/RedisClientProfileTests.cs index fbe8d2671f..cf19903e31 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Redis/RedisClientProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Redis/RedisClientProfileTests.cs @@ -8,6 +8,7 @@ namespace VirtualClient.Actions using System.Linq; using System.Net; using System.Runtime.InteropServices; + using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using NUnit.Framework; @@ -36,7 +37,9 @@ public void SetupFixture() new ClientInstance(this.clientAgentId, "1.2.3.4", "Client"), new ClientInstance(this.serverAgentId, "1.2.3.5", "Server")); - this.mockFixture.SetupPackage("wget", expectedFiles: "linux-x64/wget2"); + this.mockFixture.SetupPackage("wget", null, "linux-x64/wget2"); + this.mockFixture.SetupPackage("redis", null, "src/redis-benchmark", "src/redis-server"); + this.mockFixture.SetupPackage("memtier", null, "memtier_benchmark"); } [Test] @@ -53,46 +56,26 @@ public void RedisMemtierWorkloadProfileActionsWillNotBeExecutedIfTheClientWorklo } [Test] - [Ignore("We need to completely refactor the functional tests for Memcached and Redis to consolidate and cleanup.")] [TestCase("PERF-REDIS.json")] public async Task RedisMemtierWorkloadProfileExecutesTheWorkloadAsExpectedOfClientOnUnixPlatform(string profile) { - IEnumerable expectedCommands = new List - { - $"--protocol redis --clients 1 --threads 4 --ratio 1:9 --data-size 32 --pipeline 32 --key-minimum 1 --key-maximum 10000000 --key-pattern R:R --run-count 1 --test-time 180 --print-percentile 50,90,95,99,99.9 --random-data", - $" -h 1.2.3.5 -p 6379 -c 1 -n 10000 -P 32 -q --csv\"" - }; - - // Setup the expectations for the workload - // - Workload package is installed and exists. - // - Workload binaries/executables exist on the file system. - // - Expected processes are executed. - this.mockFixture.SetupFile(@"/home/user/tools/VirtualClient/scripts/Redis/RunClient.sh"); - this.mockFixture.SetupFile(@"/home/user/tools/VirtualClient/packages/redis-6.2.1/src/redis-benchmark"); + this.mockFixture + .TrackProcesses() + .SetupProcessOutput( + "memtier_benchmark.*--server", + TestDependencies.GetResourceFileContents("Results_RedisMemtier.txt")); this.SetupApiClient(this.serverAgentId, serverIPAddress: "1.2.3.5"); - this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => - { - IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); - - if (arguments.Contains("memtier_benchmark --server", StringComparison.OrdinalIgnoreCase)) - { - process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_RedisMemtier.txt")); - } - else if (arguments.Contains("redis-benchmark", StringComparison.OrdinalIgnoreCase)) - { - process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_RedisBenchmark.txt")); - } - - return process; - }; - using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None) .ConfigureAwait(false); - WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + + this.mockFixture.Tracking.AssertCommandsExecuted( + "sudo chmod \\+x.*memtier_benchmark", + "memtier_benchmark.*--server 1\\.2\\.3\\.5.*--port 6379.*--protocol redis", + "memtier_benchmark.*--server 1\\.2\\.3\\.5.*--port 6380.*--protocol redis"); } } diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Redis/RedisServerProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Redis/RedisServerProfileTests.cs index 3c8b6a6a2b..e535dd6ebe 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Redis/RedisServerProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Redis/RedisServerProfileTests.cs @@ -8,7 +8,7 @@ namespace VirtualClient.Actions using System.Linq; using System.Net; using System.Runtime.InteropServices; - using System.Text; + using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Moq; @@ -35,7 +35,7 @@ public void SetupFixture() ComponentTypeCache.Instance.LoadComponentTypes(TestDependencies.TestDirectory); - this.mockFixture.SetupPackage("wget", expectedFiles: "linux-x64/wget2"); + this.mockFixture.SetupPackage("wget", null, "linux-x64/wget2"); this.mockFixture.SetupFile("redis", "redis-6.2.1/src/redis-server", new byte[0]); this.mockFixture.SystemManagement.Setup(mgr => mgr.GetCpuInfoAsync(It.IsAny())) .ReturnsAsync(new CpuInfo("AnyName", "AnyDescription", 1, 4, 1, 0, true)); @@ -45,31 +45,20 @@ public void SetupFixture() [TestCase("PERF-REDIS.json")] public async Task RedisMemtierWorkloadProfileInstallsTheExpectedDependenciesOfServerOnUnixPlatform(string profile) { - using var memoryProcess = new InMemoryProcess - { - StandardOutput = new ConcurrentBuffer( - new StringBuilder("Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123")), - OnStart = () => true, - OnHasExited = () => true - }; - - this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => - { - IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); - if (arguments?.Contains("redis-server") == true && arguments?.Contains("--version") == true) - { - return memoryProcess; - } + this.mockFixture + .TrackProcesses() + .SetupProcessOutput( + "redis-server.*--version", + "Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123"); - return process; - }; using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None) .ConfigureAwait(false); - // Workload dependency package expectations WorkloadAssert.WorkloadPackageInstalled(this.mockFixture, "redis"); + + this.mockFixture.Tracking.AssertCommandsExecuted("redis-server.*--version"); } } @@ -77,16 +66,12 @@ await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None [TestCase("PERF-REDIS.json")] public async Task RedisMemtierWorkloadProfileExecutesTheWorkloadAsExpectedOfServerOnUnixPlatformMultiVM(string profile) { - List expectedCommands = new List(); - - int port = 6379; - Enumerable.Range(0, 4).ToList().ForEach(core => - expectedCommands.Add($"sudo bash -c \"numactl -C {core} /.+/redis-server --port {port + core} --protected-mode no --io-threads 4 --maxmemory-policy noeviction --ignore-warnings ARM64-COW-BUG --save &\"")); + this.mockFixture + .TrackProcesses() + .SetupProcessOutput( + "redis-server.*--version", + "Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123"); - // Setup the expectations for the workload - // - Workload package is installed and exists. - // - Workload binaries/executables exist on the file system. - // - Expected processes are executed. IPAddress.TryParse("1.2.3.5", out IPAddress ipAddress); IApiClient apiClient = this.mockFixture.ApiClientManager.GetOrCreateApiClient("1.2.3.5", ipAddress); @@ -105,29 +90,20 @@ public async Task RedisMemtierWorkloadProfileExecutesTheWorkloadAsExpectedOfServ }); await apiClient.CreateStateAsync(nameof(ServerState), state, CancellationToken.None); - using var memoryProcess = new InMemoryProcess - { - StandardOutput = new ConcurrentBuffer( - new StringBuilder("Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123")), - OnStart = () => true, - OnHasExited = () => true - }; - - this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => - { - IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); - if (arguments?.Contains("redis-server") == true && arguments?.Contains("--version") == true) - { - return memoryProcess; - } - - return process; - }; using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None); - WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + + this.mockFixture.Tracking.AssertCommandsExecuted(true, + $"sudo chmod \\+x.*redis-server", + $"sudo bash -c \\\"numactl -C 0.*redis-server --port 6379.*\\\"", + $"sudo bash -c \\\"numactl -C 1.*redis-server --port 6380.*\\\"", + $"sudo bash -c \\\"numactl -C 2.*redis-server --port 6381.*\\\"", + $"sudo bash -c \\\"numactl -C 3.*redis-server --port 6382.*\\\""); + + this.mockFixture.Tracking.AssertCommandExecutedTimes("chmod.*redis-server", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("numactl.*redis-server", 4); } } } diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisBenchmarkClientExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisBenchmarkClientExecutorTests.cs index 2994d0b281..c95a0a2598 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisBenchmarkClientExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisBenchmarkClientExecutorTests.cs @@ -8,6 +8,7 @@ namespace VirtualClient.Actions using System.IO; using System.Net; using System.Net.Http; + using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -96,36 +97,30 @@ public async Task RedisBenchmarkClientExecutorOnInitializationGetsTheExpectedPac [Test] public async Task RedisBenchmarkClientExecutorExecutesExpectedCommands() { + this.fixture + .TrackProcesses() + .SetupProcessOutput(".*", this.results); + using (var executor = new TestRedisBenchmarkClientExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { - List expectedCommands = new List() - { - // Make the benchmark toolset executable - $"sudo chmod +x \"{this.mockPackage.Path}/src/redis-benchmark\"", - - // Run the Redis benchmark. Values based on the default parameter values set at the top - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 6379 {executor.CommandLine}\"" - }; - - this.fixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDirectory) => - { - expectedCommands.Remove($"{exe} {arguments}"); - this.fixture.Process.StandardOutput.Append(this.results); - - return this.fixture.Process; - }; - await executor.ExecuteAsync(CancellationToken.None); - Assert.IsEmpty(expectedCommands); + + this.fixture.Tracking.AssertCommandsExecuted(true, + $"sudo chmod \\+x \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark\\\"", + $"sudo bash -c \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark -h 1.2.3.5 -p 6379 {Regex.Escape(executor.CommandLine)}\\\"" + ); } } [Test] public async Task RedisBenchmarkClientExecutorEstablishesTheExpectedClientServerPairings_1_Server_Instance() { + this.fixture + .TrackProcesses() + .SetupProcessOutput(".*", this.results); + using (var executor = new TestRedisBenchmarkClientExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { - // 1 set of client instances run on each port reported by the server ServerState serverState = new ServerState(new List { new PortDescription @@ -138,31 +133,24 @@ public async Task RedisBenchmarkClientExecutorEstablishesTheExpectedClientServer this.fixture.ApiClient.OnGetState(nameof(ServerState)) .ReturnsAsync(this.fixture.CreateHttpResponse(HttpStatusCode.OK, new Item(nameof(ServerState), serverState))); - List expectedCommands = new List() - { - // 1 client instances for the server on port 1234 - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 1234 {executor.CommandLine}\"", - }; - - this.fixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDirectory) => - { - expectedCommands.Remove($"{exe} {arguments}"); - this.fixture.Process.StandardOutput.Append(this.results); - - return this.fixture.Process; - }; - await executor.ExecuteAsync(CancellationToken.None); - Assert.IsEmpty(expectedCommands); + + this.fixture.Tracking.AssertCommandsExecuted(true, + $"sudo chmod \\+x \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark\\\"", + $"sudo bash -c \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark -h 1.2.3.5 -p 1234 {Regex.Escape(executor.CommandLine)}\\\"" + ); } } [Test] public async Task RedisBenchmarkClientExecutorEstablishesTheExpectedClientServerPairings_2_Server_Instances() { + this.fixture + .TrackProcesses() + .SetupProcessOutput(".*", this.results); + using (var executor = new TestRedisBenchmarkClientExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { - // 1 set of client instances run on each port reported by the server ServerState serverState = new ServerState(new List { new PortDescription @@ -180,164 +168,92 @@ public async Task RedisBenchmarkClientExecutorEstablishesTheExpectedClientServer this.fixture.ApiClient.OnGetState(nameof(ServerState)) .ReturnsAsync(this.fixture.CreateHttpResponse(HttpStatusCode.OK, new Item(nameof(ServerState), serverState))); - List expectedCommands = new List() - { - // 1 client instances for the server on port 1234 - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 1234 {executor.CommandLine}\"", - - // 1 client instances for the server on port 5678 - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 5678 {executor.CommandLine}\"" - }; - - this.fixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDirectory) => - { - expectedCommands.Remove($"{exe} {arguments}"); - this.fixture.Process.StandardOutput.Append(this.results); - - return this.fixture.Process; - }; - await executor.ExecuteAsync(CancellationToken.None); - Assert.IsEmpty(expectedCommands); + + this.fixture.Tracking.AssertCommandsExecuted(true, + $"sudo chmod \\+x \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark\\\"", + $"sudo bash -c \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark -h 1.2.3.5 -p 1234 {Regex.Escape(executor.CommandLine)}\\\"", + $"sudo bash -c \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark -h 1.2.3.5 -p 5678 {Regex.Escape(executor.CommandLine)}\\\"" + ); } } [Test] public async Task RedisBenchmarkClientExecutorEstablishesTheExpectedClientServerPairings_4_Server_Instances() { + this.fixture + .TrackProcesses() + .SetupProcessOutput(".*", this.results); + using (var executor = new TestRedisBenchmarkClientExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { - // 1 set of client instances run on each port reported by the server ServerState serverState = new ServerState(new List { - new PortDescription - { - CpuAffinity = "0", - Port = 1234 - }, - new PortDescription - { - CpuAffinity = "1", - Port = 5678 - }, - new PortDescription - { - CpuAffinity = "2", - Port = 1111 - }, - new PortDescription - { - CpuAffinity = "3", - Port = 2222 - } + new PortDescription { CpuAffinity = "0", Port = 1234 }, + new PortDescription { CpuAffinity = "1", Port = 5678 }, + new PortDescription { CpuAffinity = "2", Port = 1111 }, + new PortDescription { CpuAffinity = "3", Port = 2222 } }); this.fixture.ApiClient.OnGetState(nameof(ServerState)) .ReturnsAsync(this.fixture.CreateHttpResponse(HttpStatusCode.OK, new Item(nameof(ServerState), serverState))); - List expectedCommands = new List() - { - // 1 client instances for the server on port 1234 - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 1234 {executor.CommandLine}\"", - - // 1 client instances for the server on port 5678 - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 5678 {executor.CommandLine}\"", - - // 1 client instances for the server on port 1234 - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 1111 {executor.CommandLine}\"", - - // 1 client instances for the server on port 5678 - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 2222 {executor.CommandLine}\"" - }; - - this.fixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDirectory) => - { - expectedCommands.Remove($"{exe} {arguments}"); - this.fixture.Process.StandardOutput.Append(this.results); - - return this.fixture.Process; - }; - await executor.ExecuteAsync(CancellationToken.None); - Assert.IsEmpty(expectedCommands); + + this.fixture.Tracking.AssertCommandsExecuted(true, + $"sudo chmod \\+x \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark\\\"", + $"sudo bash -c \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark -h 1.2.3.5 -p 1234 {Regex.Escape(executor.CommandLine)}\\\"", + $"sudo bash -c \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark -h 1.2.3.5 -p 5678 {Regex.Escape(executor.CommandLine)}\\\"", + $"sudo bash -c \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark -h 1.2.3.5 -p 1111 {Regex.Escape(executor.CommandLine)}\\\"", + $"sudo bash -c \\\"{Regex.Escape(this.mockPackage.Path)}/src/redis-benchmark -h 1.2.3.5 -p 2222 {Regex.Escape(executor.CommandLine)}\\\"" + ); } } [Test] public async Task RedisBenchmarkClientExecutorEstablishesTheExpectedClientServerPairings_1_Server_Instance_2_Client_Instances() { + this.fixture + .TrackProcesses() + .SetupProcessOutput(".*", this.results); + using (var executor = new TestRedisBenchmarkClientExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { - // 2 client instances per server instance executor.Parameters["ClientInstances"] = 2; - List expectedCommands = new List() - { - // 2 client instances for the server on port 6379 (defined in default setup). - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 6379 {executor.CommandLine}\"", - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 6379 {executor.CommandLine}\"", - }; - - this.fixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDirectory) => - { - expectedCommands.Remove($"{exe} {arguments}"); - this.fixture.Process.StandardOutput.Append(this.results); - - return this.fixture.Process; - }; - await executor.ExecuteAsync(CancellationToken.None); - Assert.IsEmpty(expectedCommands); + + this.fixture.Tracking.AssertCommandExecutedTimes( + $"sudo bash -c.*redis-benchmark -h 1.2.3.5 -p 6379", + 2 + ); } } [Test] public async Task RedisBenchmarkClientExecutorEstablishesTheExpectedClientServerPairings_2_Server_Instances_2_Client_Instances() { + this.fixture + .TrackProcesses() + .SetupProcessOutput(".*", this.results); + using (var executor = new TestRedisBenchmarkClientExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { - // 2 client instances per server instance executor.Parameters["ClientInstances"] = 2; - // 1 set of client instances run on each port reported by the server ServerState serverState = new ServerState(new List { - new PortDescription - { - CpuAffinity = "0", - Port = 1234 - }, - new PortDescription - { - CpuAffinity = "1", - Port = 5678 - } + new PortDescription { CpuAffinity = "0", Port = 1234 }, + new PortDescription { CpuAffinity = "1", Port = 5678 } }); this.fixture.ApiClient.OnGetState(nameof(ServerState)) .ReturnsAsync(this.fixture.CreateHttpResponse(HttpStatusCode.OK, new Item(nameof(ServerState), serverState))); - List expectedCommands = new List() - { - // 2 client instances for the server on port 1234. - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 1234 {executor.CommandLine}\"", - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 1234 {executor.CommandLine}\"", - - // 2 client instances for the server on port 5678. - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 5678 {executor.CommandLine}\"", - $"sudo bash -c \"{this.mockPackage.Path}/src/redis-benchmark -h 1.2.3.5 -p 5678 {executor.CommandLine}\"", - }; - - this.fixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDirectory) => - { - expectedCommands.Remove($"{exe} {arguments}"); - this.fixture.Process.StandardOutput.Append(this.results); - - return this.fixture.Process; - }; - await executor.ExecuteAsync(CancellationToken.None); - Assert.IsEmpty(expectedCommands); + + this.fixture.Tracking.AssertCommandExecutedTimes("redis-benchmark -h 1.2.3.5 -p 1234", 2); + this.fixture.Tracking.AssertCommandExecutedTimes("redis-benchmark -h 1.2.3.5 -p 5678", 2); } } diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisServerExecutorTests.cs index fa062f6097..c085431fbf 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisServerExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisServerExecutorTests.cs @@ -8,6 +8,7 @@ namespace VirtualClient.Actions using System.Linq; using System.Net; using System.Text; + using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -69,19 +70,12 @@ public void SetupTests() [Test] public async Task RedisServerExecutorConfirmsTheExpectedPackagesOnInitialization() { + this.fixture.SetupProcessOutput( + "redis-server.*--version", + "Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123"); + using (var component = new TestRedisServerExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { - this.fixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => - { - if (arguments?.Contains("redis-server") == true && arguments?.Contains("--version") == true) - { - this.memoryProcess.StandardOutput = new ConcurrentBuffer( - new StringBuilder("Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123") - ); - return this.memoryProcess; - } - return this.memoryProcess; - }; await component.InitializeAsync(EventContext.None, CancellationToken.None); this.fixture.PackageManager.Verify(mgr => mgr.GetPackageAsync(this.mockRedisPackage.Name, It.IsAny())); } @@ -90,125 +84,91 @@ public async Task RedisServerExecutorConfirmsTheExpectedPackagesOnInitialization [Test] public async Task RedisMemtierServerExecutorExecutesExpectedProcessWhenBindingToCores() { + this.fixture + .TrackProcesses() + .SetupProcessOutput( + "redis-server.*--version", + "Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123"); + using (var executor = new TestRedisServerExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { - List expectedCommands = new List() - { - // Make the Redis server toolset executable - $"sudo chmod +x \"{this.mockRedisPackage.Path}/src/redis-server\"", - - // Start the server binded to the logical core. Values based on the parameters set at the top. - $"sudo bash -c \"numactl -C 0 {this.mockRedisPackage.Path}/src/redis-server --port 6379 --protected-mode no --io-threads 4 --maxmemory-policy noeviction --ignore-warnings ARM64-COW-BUG --save --daemonize yes\"" - }; - - this.fixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDirectory) => - { - if (arguments?.Contains("redis-server") == true && arguments?.Contains("--version") == true) - { - this.memoryProcess.StandardOutput = new ConcurrentBuffer( - new StringBuilder("Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123") - ); - return this.memoryProcess; - } - expectedCommands.Remove($"{exe} {arguments}"); - return this.fixture.Process; - }; - await executor.ExecuteAsync(CancellationToken.None); - Assert.IsEmpty(expectedCommands); + this.fixture.Tracking.AssertCommandsExecuted(true, + $"sudo chmod \\+x \\\"{Regex.Escape(this.mockRedisPackage.Path)}/src/redis-server\\\"", + $"sudo bash -c \\\"numactl -C 0 {Regex.Escape(this.mockRedisPackage.Path)}/src/redis-server --port 6379 --protected-mode no --io-threads 4 --maxmemory-policy noeviction --ignore-warnings ARM64-COW-BUG --save --daemonize yes\\\"" + ); + + this.fixture.Tracking.AssertCommandExecutedTimes("chmod", 1); + this.fixture.Tracking.AssertCommandExecutedTimes("numactl", 1); } } [Test] public async Task RedisMemtierServerExecutorExecutesExpectedProcessWhenBindingToCores_2_Server_Instances() { + this.fixture + .TrackProcesses() + .SetupProcessOutput( + "redis-server.*--version", + "Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123"); + using (var executor = new TestRedisServerExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { executor.Parameters[nameof(executor.ServerInstances)] = 2; - List expectedCommands = new List() - { - // Make the Redis server toolset executable - $"sudo chmod +x \"{this.mockRedisPackage.Path}/src/redis-server\"", + await executor.ExecuteAsync(CancellationToken.None); + this.fixture.Tracking.AssertCommandsExecuted(true, + // Make redis-server executable + $"sudo chmod \\+x \\\"{Regex.Escape(this.mockRedisPackage.Path)}/src/redis-server\\\"", + // Server instance #1 bound to core 0 and running on port 6379 - $"sudo bash -c \"numactl -C 0 {this.mockRedisPackage.Path}/src/redis-server --port 6379 --protected-mode no --io-threads 4 --maxmemory-policy noeviction --ignore-warnings ARM64-COW-BUG --save --daemonize yes\"", - + $"sudo bash -c \\\"numactl -C 0 {Regex.Escape(this.mockRedisPackage.Path)}/src/redis-server --port 6379 --protected-mode no --io-threads 4 --maxmemory-policy noeviction --ignore-warnings ARM64-COW-BUG --save --daemonize yes\\\"", + // Server instance #2 bound to core 1 and running on port 6380 - $"sudo bash -c \"numactl -C 1 {this.mockRedisPackage.Path}/src/redis-server --port 6380 --protected-mode no --io-threads 4 --maxmemory-policy noeviction --ignore-warnings ARM64-COW-BUG --save --daemonize yes\"" - }; + $"sudo bash -c \\\"numactl -C 1 {Regex.Escape(this.mockRedisPackage.Path)}/src/redis-server --port 6380 --protected-mode no --io-threads 4 --maxmemory-policy noeviction --ignore-warnings ARM64-COW-BUG --save --daemonize yes\\\"" + ); - this.fixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDirectory) => - { - if (arguments?.Contains("redis-server") == true && arguments?.Contains("--version") == true) - { - this.memoryProcess.StandardOutput = new ConcurrentBuffer( - new StringBuilder("Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123") - ); - return this.memoryProcess; - } - expectedCommands.Remove($"{exe} {arguments}"); - return this.fixture.Process; - }; - - await executor.ExecuteAsync(CancellationToken.None); - Assert.IsEmpty(expectedCommands); + // Verify 2 numactl commands executed (one per instance) + this.fixture.Tracking.AssertCommandExecutedTimes("numactl", 2); } } [Test] public async Task RedisMemtierServerExecutorExecutesExpectedProcessWhenNotBindingToCores() { + this.fixture + .TrackProcesses() + .SetupProcessOutput( + "redis-server.*--version", + "Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123"); + using (var executor = new TestRedisServerExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { executor.Parameters[nameof(executor.BindToCores)] = false; - List expectedCommands = new List() - { - // Make the Redis server toolset executable - $"sudo chmod +x \"{this.mockRedisPackage.Path}/src/redis-server\"", - - // Start the server binded to the logical core. Values based on the parameters set at the top. - $"sudo bash -c \"{this.mockRedisPackage.Path}/src/redis-server --port 6379 --protected-mode no --io-threads 4 --maxmemory-policy noeviction --ignore-warnings ARM64-COW-BUG --save --daemonize yes\"" - }; + await executor.ExecuteAsync(CancellationToken.None); - this.fixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDirectory) => - { - if (arguments?.Contains("redis-server") == true && arguments?.Contains("--version") == true) - { - this.memoryProcess.StandardOutput = new ConcurrentBuffer( - new StringBuilder("Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123") - ); - return this.memoryProcess; - } - expectedCommands.Remove($"{exe} {arguments}"); - return this.fixture.Process; - }; + this.fixture.Tracking.AssertCommandsExecuted(true, + $"sudo chmod \\+x \\\"{Regex.Escape(this.mockRedisPackage.Path)}/src/redis-server\\\"", + $"{Regex.Escape(this.mockRedisPackage.Path)}/src/redis-server --port 6379 --protected-mode no --io-threads 4 --maxmemory-policy noeviction --ignore-warnings ARM64-COW-BUG --save --daemonize yes" + ); - await executor.ExecuteAsync(CancellationToken.None); - Assert.IsEmpty(expectedCommands); + this.fixture.Tracking.AssertCommandExecutedTimes("numactl", 0); } } [Test] public async Task RedisServerExecutorCapturesRedisVersionSuccessfully() { + this.fixture.SetupProcessOutput( + "redis-server.*--version", + "Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123"); + using (var executor = new TestRedisServerExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { - this.fixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => - { - if (arguments?.Contains("redis-server") == true && arguments?.Contains("--version") == true) - { - this.memoryProcess.StandardOutput = new ConcurrentBuffer( - new StringBuilder("Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123") - ); - return this.memoryProcess; - } - return this.memoryProcess; - }; - // Act await executor.InitializeAsync(EventContext.None, CancellationToken.None); - // Assert + var messages = this.fixture.Logger.MessagesLogged($"{nameof(TestRedisServerExecutor)}.RedisVersionCaptured"); Assert.IsNotEmpty(messages, "Expected at least one log message indicating the Redis version was captured."); bool versionCapturedCorrectly = messages.Any(msg => diff --git a/src/VirtualClient/VirtualClient.TestFramework.UnitTests/FixtureTrackingTests.cs b/src/VirtualClient/VirtualClient.TestFramework.UnitTests/FixtureTrackingTests.cs new file mode 100644 index 0000000000..f1c5986e7c --- /dev/null +++ b/src/VirtualClient/VirtualClient.TestFramework.UnitTests/FixtureTrackingTests.cs @@ -0,0 +1,293 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient +{ + using System; + using System.Diagnostics; + using NUnit.Framework; + using VirtualClient.Common; + + [TestFixture] + [Category("Unit")] + internal class FixtureTrackingTests + { + private FixtureTracking tracking; + + [SetUp] + public void SetupTest() + { + this.tracking = new FixtureTracking(); + } + + [Test] + public void AssertCommandsExecutedThrowsWhenNoCommandsWereExecuted() + { + Assert.Throws(() => this.tracking.AssertCommandsExecuted("any-command")); + } + + [Test] + public void AssertCommandsExecutedThrowsWhenExpectedCommandWasNotExecuted() + { + this.AddCommand("command1 --arg1"); + + Assert.Throws(() => this.tracking.AssertCommandsExecuted("command2")); + } + + [Test] + public void AssertCommandsExecutedDoesNotThrowWhenExpectedCommandWasExecuted() + { + this.AddCommand("command1 --arg1"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandsExecuted("command1 --arg1")); + } + + [Test] + public void AssertCommandsExecutedMatchesUsingRegularExpressions() + { + this.AddCommand("/home/user/bin/workload --threads 4 --size 128"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandsExecuted("workload.*--threads 4")); + } + + [Test] + public void AssertCommandsExecutedMatchesRegardlessOfOrder() + { + this.AddCommand("command1"); + this.AddCommand("command2"); + this.AddCommand("command3"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandsExecuted("command3", "command1")); + } + + [Test] + public void AssertCommandsExecutedMatchesAreCaseInsensitive() + { + this.AddCommand("SomeCommand --Arg1"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandsExecuted("somecommand --arg1")); + } + + [Test] + public void AssertCommandsExecutedFallsBackToExactMatchWhenRegexIsInvalid() + { + // The '[' is an invalid regex pattern by itself. + this.AddCommand("[invalid-regex"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandsExecuted("[invalid-regex")); + } + + [Test] + public void AssertCommandsExecutedDoesNotMatchTheSameCommandTwice() + { + this.AddCommand("command1"); + + Assert.Throws(() => this.tracking.AssertCommandsExecuted("command1", "command1")); + } + + [Test] + public void AssertCommandsExecutedInOrderThrowsWhenNoCommandsWereExecuted() + { + Assert.Throws(() => this.tracking.AssertCommandsExecuted(true, "any-command")); + } + + [Test] + public void AssertCommandsExecutedInOrderThrowsWhenCommandsAreOutOfOrder() + { + this.AddCommand("command1"); + this.AddCommand("command2"); + + Assert.Throws(() => this.tracking.AssertCommandsExecuted(true, "command2", "command1")); + } + + [Test] + public void AssertCommandsExecutedInOrderDoesNotThrowWhenCommandsAreInOrder() + { + this.AddCommand("command1"); + this.AddCommand("command2"); + this.AddCommand("command3"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandsExecuted(true, "command1", "command3")); + } + + [Test] + public void AssertCommandsExecutedInOrderMatchesUsingRegularExpressions() + { + this.AddCommand("sudo chmod +x /home/user/workload"); + this.AddCommand("sudo /home/user/workload --port 6379"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandsExecuted(true, + "chmod.*workload", + "workload.*--port 6379")); + } + + [Test] + public void AssertCommandsExecutedInOrderFallsBackToExactMatchWhenRegexIsInvalid() + { + this.AddCommand("[first"); + this.AddCommand("[second"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandsExecuted(true, "[first", "[second")); + } + + [Test] + public void AssertCommandExecutedTimesThrowsWhenCountDoesNotMatch() + { + this.AddCommand("command1"); + + Assert.Throws(() => this.tracking.AssertCommandExecutedTimes("command1", 2)); + } + + [Test] + public void AssertCommandExecutedTimesDoesNotThrowWhenCountMatches() + { + this.AddCommand("command1 --port 6379"); + this.AddCommand("command1 --port 6380"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandExecutedTimes("command1", 2)); + } + + [Test] + public void AssertCommandExecutedTimesHandlesZeroExpectedExecutions() + { + this.AddCommand("command1"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandExecutedTimes("command2", 0)); + } + + [Test] + public void AssertCommandExecutedTimesMatchesUsingRegularExpressions() + { + this.AddCommand("numactl -C 0 redis-server --port 6379"); + this.AddCommand("numactl -C 1 redis-server --port 6380"); + this.AddCommand("chmod +x redis-server"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandExecutedTimes("numactl.*redis-server", 2)); + } + + [Test] + public void AssertCommandExecutedTimesFallsBackToSubstringMatchWhenRegexIsInvalid() + { + this.AddCommand("run [test"); + this.AddCommand("run [test"); + + Assert.DoesNotThrow(() => this.tracking.AssertCommandExecutedTimes("[test", 2)); + } + + [Test] + public void ClearRemovesAllTrackedCommands() + { + this.AddCommand("command1"); + this.AddCommand("command2"); + + this.tracking.Clear(); + + Assert.AreEqual(0, this.tracking.Commands.Count); + Assert.Throws(() => this.tracking.AssertCommandsExecuted("command1")); + } + + [Test] + public void CommandsPropertyReturnsAllTrackedCommandsInOrder() + { + this.AddCommand("first"); + this.AddCommand("second"); + this.AddCommand("third"); + + Assert.AreEqual(3, this.tracking.Commands.Count); + Assert.AreEqual("first", this.tracking.Commands[0].FullCommand); + Assert.AreEqual("second", this.tracking.Commands[1].FullCommand); + Assert.AreEqual("third", this.tracking.Commands[2].FullCommand); + } + + [Test] + public void GetDetailedSummaryReturnsFormattedSummaryOfTrackedCommands() + { + this.AddCommand("command1 --arg1"); + + string summary = this.tracking.GetDetailedSummary(); + + Assert.IsNotNull(summary); + StringAssert.Contains("Total Commands Executed: 1", summary); + StringAssert.Contains("command1 --arg1", summary); + } + + [Test] + public void GetDetailedSummaryHandlesNoCommands() + { + string summary = this.tracking.GetDetailedSummary(); + + Assert.IsNotNull(summary); + StringAssert.Contains("Total Commands Executed: 0", summary); + } + + [Test] + public void ErrorMessageIncludesActualCommandsExecuted() + { + this.AddCommand("actual-command --flag"); + + InvalidOperationException error = Assert.Throws( + () => this.tracking.AssertCommandsExecuted("missing-command")); + + StringAssert.Contains("actual-command --flag", error.Message); + StringAssert.Contains("Missing Commands:", error.Message); + } + + [Test] + public void ErrorMessageForOrderedAssertionIncludesExpectedAndActualOrder() + { + this.AddCommand("command2"); + this.AddCommand("command1"); + + InvalidOperationException error = Assert.Throws( + () => this.tracking.AssertCommandsExecuted(true, "command1", "command2")); + + StringAssert.Contains("Expected Order:", error.Message); + StringAssert.Contains("Actual Execution Order:", error.Message); + } + + [Test] + public void ErrorMessageForCountMismatchIncludesExpectedAndActualCounts() + { + this.AddCommand("command1"); + + InvalidOperationException error = Assert.Throws( + () => this.tracking.AssertCommandExecutedTimes("command1", 3)); + + StringAssert.Contains("Expected: 3 execution(s)", error.Message); + StringAssert.Contains("Actual: 1 execution(s)", error.Message); + } + + private void AddCommand(string fullCommand) + { + // Split on first space to separate command from arguments, matching how + // InMemoryProcessManager records executions. + string command = fullCommand; + string arguments = null; + + int spaceIndex = fullCommand.IndexOf(' '); + if (spaceIndex >= 0) + { + command = fullCommand.Substring(0, spaceIndex); + arguments = fullCommand.Substring(spaceIndex + 1); + } + + InMemoryProcess process = new InMemoryProcess + { + StartInfo = new ProcessStartInfo + { + FileName = command, + Arguments = arguments + }, + OnHasExited = () => true, + OnStart = () => true + }; + + this.tracking.AddCommand(new CommandExecutionInfo( + command, + arguments, + null, + process, + DateTime.UtcNow)); + } + } +} diff --git a/src/VirtualClient/VirtualClient.TestFramework/CommandExecutionInfo.cs b/src/VirtualClient/VirtualClient.TestFramework/CommandExecutionInfo.cs new file mode 100644 index 0000000000..6862335b93 --- /dev/null +++ b/src/VirtualClient/VirtualClient.TestFramework/CommandExecutionInfo.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient +{ + using System; + using VirtualClient.Common; + + /// + /// Represents information about a command/process execution captured during testing. + /// + public class CommandExecutionInfo + { + /// + /// Initializes a new instance of the class. + /// + /// The command/executable that was run. + /// The command-line arguments. + /// The working directory for the command execution. + /// The process proxy associated with the execution. + /// The time at which the command was executed. + public CommandExecutionInfo( + string command, + string arguments, + string workingDirectory, + IProcessProxy process, + DateTime executionTime) + { + this.Command = command; + this.Arguments = arguments; + this.WorkingDirectory = workingDirectory; + this.Process = process; + this.ExecutionTime = executionTime; + } + + /// + /// The command/executable that was run. + /// + public string Command { get; } + + /// + /// The command-line arguments. + /// + public string Arguments { get; } + + /// + /// The working directory for the command execution. + /// + public string WorkingDirectory { get; } + + /// + /// The full command string (command + arguments). + /// + public string FullCommand => string.IsNullOrEmpty(this.Arguments) + ? this.Command + : $"{this.Command} {this.Arguments}"; + + /// + /// The process proxy associated with the execution. + /// + public IProcessProxy Process { get; } + + /// + /// The time at which the command was executed. + /// + public DateTime ExecutionTime { get; } + + /// + /// Exit code of the process. + /// + public int ExitCode => this.Process?.ExitCode ?? 0; + + /// + /// Standard output from the process. + /// + public string StandardOutput => this.Process?.StandardOutput?.ToString(); + + /// + /// Standard error from the process. + /// + public string StandardError => this.Process?.StandardError?.ToString(); + } +} diff --git a/src/VirtualClient/VirtualClient.TestFramework/DependencyFixture.cs b/src/VirtualClient/VirtualClient.TestFramework/DependencyFixture.cs index d0da87f19a..018791be9b 100644 --- a/src/VirtualClient/VirtualClient.TestFramework/DependencyFixture.cs +++ b/src/VirtualClient/VirtualClient.TestFramework/DependencyFixture.cs @@ -202,6 +202,11 @@ public string PlatformArchitectureName /// public ProfileTiming Timing { get; set; } + /// + /// Gets the process tracking instance. This is populated after the method is called. + /// + public FixtureTracking Tracking => this.ProcessManager.Tracking; + /// /// Returns a path that is combined specific to the platform defined for this /// fixture. @@ -405,6 +410,36 @@ public DependencyPath ToPlatformSpecificPath(DependencyPath dependency, Platform return this.PlatformSpecifics.ToPlatformSpecificPath(dependency, platform, architecture); } + /// + /// Enables automatic tracking of all process executions. + /// + /// True to clear any previously tracked commands. + /// The fixture instance for method chaining. + public DependencyFixture TrackProcesses(bool reset = true) + { + this.ProcessManager.TrackProcesses(reset); + return this; + } + + /// + /// Sets up automatic output for processes whose full command line matches + /// the pattern provided. + /// + /// A regex pattern matching the command. + /// The standard output to return for matching commands. + /// The standard error output (optional). + /// The exit code for the process (default: 0). + /// The fixture instance for method chaining. + public DependencyFixture SetupProcessOutput( + string commandPattern, + string standardOutput, + string standardError = null, + int exitCode = 0) + { + this.ProcessManager.SetupProcessOutput(commandPattern, standardOutput, standardError, exitCode); + return this; + } + private IServiceCollection InitializeDependencies() { IServiceCollection dependencies = new ServiceCollection() diff --git a/src/VirtualClient/VirtualClient.TestFramework/FixtureTracking.cs b/src/VirtualClient/VirtualClient.TestFramework/FixtureTracking.cs new file mode 100644 index 0000000000..99133e5ab7 --- /dev/null +++ b/src/VirtualClient/VirtualClient.TestFramework/FixtureTracking.cs @@ -0,0 +1,327 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Text.RegularExpressions; + using VirtualClient.Common; + using VirtualClient.Common.Extensions; + + /// + /// Provides tracking of process/command executions and assertion methods + /// for test validation scenarios. + /// + public class FixtureTracking + { + private readonly List commands; + + /// + /// Initializes a new instance of the class. + /// + public FixtureTracking() + { + this.commands = new List(); + } + + /// + /// The set of all commands executed (in order). + /// + public IReadOnlyList Commands => this.commands.AsReadOnly(); + + /// + /// The set of all process proxies created (derived from ). + /// + public IReadOnlyList Processes => this.commands.Select(c => c.Process).ToList().AsReadOnly(); + + /// + /// Clears all tracked commands. + /// + public void Clear() + { + this.commands.Clear(); + } + + /// + /// Asserts that the specified commands were executed (in any order). Commands are + /// matched using regular expression evaluation. If the expression is not a valid + /// regular expression, an exact match (case-insensitive) comparison is used as a fallback. + /// + /// + /// The expected commands to match. Each entry can be an exact command string or a + /// regular expression pattern. + /// + public void AssertCommandsExecuted(params string[] expectedCommands) + { + this.AssertCommandsExecuted(false, expectedCommands); + } + + /// + /// Asserts that the specified commands were executed. Commands are matched using + /// regular expression evaluation. If the expression is not a valid regular expression, + /// an exact match (case-insensitive) comparison is used as a fallback. + /// + /// + /// True to require the commands were executed in the exact order specified. False to + /// allow matching in any order. + /// + /// + /// The expected commands to match. Each entry can be an exact command string or a + /// regular expression pattern. + /// + public void AssertCommandsExecuted(bool exactOrder, params string[] expectedCommands) + { + expectedCommands.ThrowIfNullOrEmpty(nameof(expectedCommands)); + + List notFound = new List(); + + if (exactOrder) + { + int currentIndex = 0; + + foreach (string pattern in expectedCommands) + { + bool found = false; + + for (int i = currentIndex; i < this.commands.Count; i++) + { + if (this.IsMatch(this.commands[i].FullCommand, pattern)) + { + currentIndex = i + 1; + found = true; + break; + } + } + + if (!found) + { + notFound.Add(pattern); + } + } + } + else + { + List matchedCommands = new List(); + + foreach (string pattern in expectedCommands) + { + CommandExecutionInfo match = this.commands.FirstOrDefault(cmd => + this.IsMatch(cmd.FullCommand, pattern) && !matchedCommands.Contains(cmd)); + + if (match == null) + { + notFound.Add(pattern); + } + else + { + matchedCommands.Add(match); + } + } + } + + if (notFound.Any()) + { + string errorMessage = this.BuildCommandNotFoundErrorMessage(notFound, expectedCommands, exactOrder); + throw new InvalidOperationException(errorMessage); + } + } + + /// + /// Asserts that a command matching the pattern was executed exactly the expected + /// number of times. + /// + /// A regular expression pattern for the command. + /// The expected number of executions. + public void AssertCommandExecutedTimes(string commandPattern, int expectedCount) + { + commandPattern.ThrowIfNullOrWhiteSpace(nameof(commandPattern)); + + int actualCount = this.commands.Count(cmd => this.IsMatch(cmd.FullCommand, commandPattern)); + + if (actualCount != expectedCount) + { + string errorMessage = this.BuildCountMismatchErrorMessage( + commandPattern, + expectedCount, + actualCount); + throw new InvalidOperationException(errorMessage); + } + } + + /// + /// Returns a detailed summary of all tracked commands. This is useful for + /// debugging test failures. + /// + public string GetDetailedSummary() + { + StringBuilder summary = new StringBuilder(); + summary.AppendLine($"Total Commands Executed: {this.commands.Count}"); + summary.AppendLine(); + + for (int i = 0; i < this.commands.Count; i++) + { + CommandExecutionInfo cmd = this.commands[i]; + summary.AppendLine($"[{i + 1}] Command: {cmd.FullCommand}"); + summary.AppendLine($" Working Dir: {cmd.WorkingDirectory}"); + summary.AppendLine($" Exit Code: {cmd.ExitCode}"); + summary.AppendLine($" Executed At: {cmd.ExecutionTime:yyyy-MM-dd HH:mm:ss.fff}"); + + if (!string.IsNullOrWhiteSpace(cmd.StandardOutput)) + { + string output = cmd.StandardOutput.Length > 200 + ? cmd.StandardOutput.Substring(0, 200) + "..." + : cmd.StandardOutput; + summary.AppendLine($" Output: {output}"); + } + + summary.AppendLine(); + } + + return summary.ToString(); + } + + /// + /// Adds a command execution record to the tracking list. + /// + internal void AddCommand(CommandExecutionInfo commandInfo) + { + commandInfo.ThrowIfNull(nameof(commandInfo)); + this.commands.Add(commandInfo); + } + + private bool IsMatch(string fullCommand, string pattern) + { + try + { + return Regex.IsMatch(fullCommand, pattern, RegexOptions.IgnoreCase); + } + catch + { + return fullCommand.IndexOf(pattern, StringComparison.OrdinalIgnoreCase) >= 0; + } + } + + private string BuildCommandNotFoundErrorMessage( + List notFound, + string[] expectedPatterns, + bool exactOrder) + { + StringBuilder message = new StringBuilder(); + + if (exactOrder) + { + message.AppendLine("Expected commands were not executed in the specified order:"); + message.AppendLine(); + + message.AppendLine("Missing or Out-of-Order Commands:"); + foreach (string pattern in notFound) + { + message.AppendLine($" - {pattern}"); + } + + message.AppendLine(); + message.AppendLine("Expected Order:"); + for (int i = 0; i < expectedPatterns.Length; i++) + { + string status = notFound.Contains(expectedPatterns[i]) ? "x" : "ok"; + message.AppendLine($" {status} [{i + 1}] {expectedPatterns[i]}"); + } + + message.AppendLine(); + message.AppendLine("Actual Execution Order:"); + if (this.commands.Any()) + { + for (int i = 0; i < this.commands.Count; i++) + { + message.AppendLine($" [{i + 1}] {this.commands[i].FullCommand}"); + } + } + else + { + message.AppendLine(" (No commands executed)"); + } + + message.AppendLine(); + message.AppendLine("Debugging Hints:"); + message.AppendLine(" - Commands must appear in the order specified"); + message.AppendLine(" - Check if intermediate commands are missing from expected list"); + message.AppendLine(" - Verify regex patterns match actual command syntax"); + } + else + { + message.AppendLine("Expected commands were not executed:"); + message.AppendLine(); + + message.AppendLine("Missing Commands:"); + foreach (string pattern in notFound) + { + message.AppendLine($" - {pattern}"); + } + + message.AppendLine(); + message.AppendLine("Actual Commands Executed:"); + if (this.commands.Any()) + { + foreach (var cmd in this.commands) + { + message.AppendLine($" - {cmd.FullCommand}"); + } + } + else + { + message.AppendLine(" (No commands executed)"); + } + + message.AppendLine(); + message.AppendLine("Debugging Hints:"); + message.AppendLine(" - Check if the command pattern uses correct regex syntax"); + message.AppendLine(" - Verify the command is actually being executed"); + message.AppendLine(" - Use GetDetailedSummary() for full command details"); + } + + return message.ToString(); + } + + private string BuildCountMismatchErrorMessage( + string pattern, + int expectedCount, + int actualCount) + { + StringBuilder message = new StringBuilder(); + message.AppendLine($"Command execution count mismatch for pattern: '{pattern}'"); + message.AppendLine(); + message.AppendLine($"Expected: {expectedCount} execution(s)"); + message.AppendLine($"Actual: {actualCount} execution(s)"); + message.AppendLine(); + + if (actualCount > 0) + { + message.AppendLine("Matching Commands Found:"); + var matches = this.commands.Where(cmd => this.IsMatch(cmd.FullCommand, pattern)); + + foreach (var match in matches) + { + message.AppendLine($" - {match.FullCommand}"); + } + } + + message.AppendLine(); + message.AppendLine("Debugging Hints:"); + if (actualCount > expectedCount) + { + message.AppendLine(" - Command was executed more times than expected"); + message.AppendLine(" - Check for duplicate execution logic"); + } + else if (actualCount < expectedCount) + { + message.AppendLine(" - Command was executed fewer times than expected"); + message.AppendLine(" - Verify all code paths are executing"); + } + + return message.ToString(); + } + } +} diff --git a/src/VirtualClient/VirtualClient.TestFramework/InMemoryProcessManager.cs b/src/VirtualClient/VirtualClient.TestFramework/InMemoryProcessManager.cs index d4d73fdbb9..fa7b85844f 100644 --- a/src/VirtualClient/VirtualClient.TestFramework/InMemoryProcessManager.cs +++ b/src/VirtualClient/VirtualClient.TestFramework/InMemoryProcessManager.cs @@ -8,7 +8,9 @@ namespace VirtualClient using System.Diagnostics; using System.IO; using System.Linq; + using System.Text.RegularExpressions; using VirtualClient.Common; + using VirtualClient.Common.Extensions; /// /// A mock/test process manager. @@ -81,31 +83,112 @@ public IEnumerable Commands /// public override PlatformID Platform { get; } - /// - public override IProcessProxy CreateProcess(string command, string arguments = null, string workingDir = null) + /// + /// Gets the process tracking instance. This is populated after the method is called. + /// + public FixtureTracking Tracking { get; private set; } + + /// + /// Enables automatic tracking of all process executions. This wraps the existing + /// handler to record each execution in a instance. + /// + /// True to clear any previously tracked commands. + public InMemoryProcessManager TrackProcesses(bool reset = true) { - IProcessProxy process = null; - if (this.OnCreateProcess != null) + if (this.Tracking == null || reset) { - process = this.OnCreateProcess?.Invoke(command, arguments, workingDir); + this.Tracking = new FixtureTracking(); } - else + + Func existingHandler = this.OnCreateProcess; + + this.OnCreateProcess = (command, arguments, workingDirectory) => { - process = new InMemoryProcess + IProcessProxy process = existingHandler != null + ? existingHandler(command, arguments, workingDirectory) + : this.CreateDefaultProcess(command, arguments, workingDirectory); + + this.Tracking.AddCommand(new CommandExecutionInfo( + command, + arguments, + workingDirectory, + process, + DateTime.UtcNow)); + + return process; + }; + + return this; + } + + /// + /// Sets up automatic output for processes whose full command line matches + /// the pattern provided. This wraps the existing handler + /// to inject standard output and error into matching processes. + /// + /// A regex pattern (or plain substring) matched against the full command. + /// The standard output to inject into matching processes. + /// The standard error to inject into matching processes (optional). + /// The exit code for matching processes (default: 0). + public InMemoryProcessManager SetupProcessOutput( + string commandPattern, + string standardOutput, + string standardError = null, + int exitCode = 0) + { + commandPattern.ThrowIfNullOrWhiteSpace(nameof(commandPattern)); + + Func existingHandler = this.OnCreateProcess; + + this.OnCreateProcess = (command, arguments, workingDirectory) => + { + IProcessProxy process = existingHandler != null + ? existingHandler(command, arguments, workingDirectory) + : this.CreateDefaultProcess(command, arguments, workingDirectory); + + string fullCommand = string.IsNullOrEmpty(arguments) + ? command + : $"{command} {arguments}"; + + bool matches; + try + { + matches = Regex.IsMatch(fullCommand, commandPattern, RegexOptions.IgnoreCase); + } + catch + { + matches = fullCommand.Contains(commandPattern, StringComparison.OrdinalIgnoreCase); + } + + if (matches && process is InMemoryProcess inMemoryProcess) { - StartInfo = new ProcessStartInfo + // Inject output/error into the existing process so that any tracking + // wrapper already holding a reference to it sees the populated buffers. + inMemoryProcess.ExitCode = exitCode; + + if (!string.IsNullOrEmpty(standardOutput)) { - FileName = command, - Arguments = arguments, - WorkingDirectory = workingDir ?? (this.Platform == PlatformID.Unix - ? Path.GetDirectoryName(command).Replace('\\', '/') - : Path.GetDirectoryName(command)) + inMemoryProcess.StandardOutput.Append(standardOutput); } - }; - (process as InMemoryProcess).OnHasExited = () => true; - (process as InMemoryProcess).OnStart = () => true; - } + if (!string.IsNullOrEmpty(standardError)) + { + inMemoryProcess.StandardError.Append(standardError); + } + } + + return process; + }; + + return this; + } + + /// + public override IProcessProxy CreateProcess(string command, string arguments = null, string workingDir = null) + { + IProcessProxy process = this.OnCreateProcess != null + ? this.OnCreateProcess.Invoke(command, arguments, workingDir) + : this.CreateDefaultProcess(command, arguments, workingDir); (this.Processes as List).Add(process); this.OnProcessCreated?.Invoke(process); @@ -128,5 +211,24 @@ public override IProcessProxy GetProcess(int processId) return process; } + + private IProcessProxy CreateDefaultProcess(string command, string arguments, string workingDirectory) + { + InMemoryProcess process = new InMemoryProcess + { + StartInfo = new ProcessStartInfo + { + FileName = command, + Arguments = arguments, + WorkingDirectory = workingDirectory ?? (this.Platform == PlatformID.Unix + ? Path.GetDirectoryName(command).Replace('\\', '/') + : Path.GetDirectoryName(command)) + }, + OnHasExited = () => true, + OnStart = () => true + }; + + return process; + } } } diff --git a/src/VirtualClient/VirtualClient.TestFramework/MockFixture.cs b/src/VirtualClient/VirtualClient.TestFramework/MockFixture.cs index 1424825520..82c4d8cfe4 100644 --- a/src/VirtualClient/VirtualClient.TestFramework/MockFixture.cs +++ b/src/VirtualClient/VirtualClient.TestFramework/MockFixture.cs @@ -257,6 +257,11 @@ public string PlatformArchitectureName /// public ProfileTiming Timing { get; set; } + /// + /// Gets the process tracking instance. This is populated after the method is called. + /// + public FixtureTracking Tracking => this.ProcessManager.Tracking; + /// /// Returns the contents of the file at the path defined by the path segments /// (e.g. [ "/home/users/examples", "toolset", "example.txt" ]) @@ -660,6 +665,36 @@ public MockFixture SetupLayout(params ClientInstance[] clients) return this; } + /// + /// Enables automatic tracking of all process executions. + /// + /// True to clear any previously tracked commands. + /// The fixture instance for method chaining. + public MockFixture TrackProcesses(bool reset = true) + { + this.ProcessManager.TrackProcesses(reset); + return this; + } + + /// + /// Sets up automatic output for processes whose full command line matches + /// the pattern provided. + /// + /// A regex pattern matching the command. + /// The standard output to return for matching commands. + /// The standard error output (optional). + /// The exit code for the process (default: 0). + /// The fixture instance for method chaining. + public MockFixture SetupProcessOutput( + string commandPattern, + string standardOutput, + string standardError = null, + int exitCode = 0) + { + this.ProcessManager.SetupProcessOutput(commandPattern, standardOutput, standardError, exitCode); + return this; + } + /// /// Returns the path for the dependency/package given a specific platform and CPU architecture. ///