Files
dotnet__skills/eng/skill-validator/tests/Check/AgentPluginTests.cs
T
Amaury Levé 36222bf32d Add first-class custom-agent evaluation coverage (#1165)
* feat(evaluation): add custom agent coverage

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): address agent review feedback

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): reject linked fixture sources

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): preserve agent result invariants

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): fail closed on agent errors

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): preserve completion regressions

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): preserve nested command quotes

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): harden native agent evidence

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): honor declared agent layout

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): resolve declared agent sources

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): secure agent path discovery

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): reject linked dependencies

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): centralize path safety checks

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): diagnose ambiguous dependencies

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): reject linked allowed roots

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): preserve skill agent isolation

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): normalize dashboard evidence

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): preserve agent gate semantics

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): fail closed on incomplete evidence

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): preserve completion evidence

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): reject overflowing durations

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): stage verified plugin skills

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): block shell network access

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): reject linked MCP config files

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): trust manual dispatch path safety

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): keep agent plugin activation diagnostic

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): count failed tool completions

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

* fix(evaluation): synchronize agent event capture

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-15 16:47:35 +02:00

784 lines
30 KiB
C#

using System.Text.Json;
using SkillValidator.Check;
using SkillValidator.Shared;
namespace SkillValidator.Tests;
public class AgentProfilerTests
{
private static AgentInfo MakeAgent(
string content,
string name = "test-agent",
string description = "Test agent description",
string fileName = "test-agent.agent.md")
{
return new AgentInfo(name, description, $"/tmp/agents/{fileName}", content, fileName);
}
public class AgentDiscoveryPathSafetyTests
{
[Fact]
public async Task PluginDiscoveryRejectsDeclaredAgentFileSymlink()
{
var root = Path.Combine(Path.GetTempPath(), $"agent-file-link-{Guid.NewGuid():N}");
var pluginRoot = Path.Combine(root, "plugin");
var agentsDir = Path.Combine(pluginRoot, "agents");
var outsideDir = Path.Combine(root, "outside");
Directory.CreateDirectory(agentsDir);
Directory.CreateDirectory(outsideDir);
File.WriteAllText(Path.Combine(pluginRoot, "plugin.json"), """
{
"name": "demo",
"version": "1.0.0",
"description": "Demo",
"agents": ["./agents/leak.agent.md"]
}
""");
var outsideAgent = Path.Combine(outsideDir, "leak.agent.md");
File.WriteAllText(outsideAgent, """
---
name: leak
description: External agent.
---
External.
""");
if (!SymlinkTestHelper.TryCreateFile(Path.Combine(agentsDir, "leak.agent.md"), outsideAgent))
{
Directory.Delete(root, true);
return;
}
try
{
Assert.Empty(await AgentDiscovery.DiscoverAgentsInPlugin(pluginRoot));
}
finally
{
Directory.Delete(root, true);
}
}
[Fact]
public async Task PluginDiscoveryRejectsDeclaredDirectorySymlink()
{
var root = Path.Combine(Path.GetTempPath(), $"agent-dir-link-{Guid.NewGuid():N}");
var pluginRoot = Path.Combine(root, "plugin");
var outsideDir = Path.Combine(root, "outside");
Directory.CreateDirectory(pluginRoot);
Directory.CreateDirectory(outsideDir);
File.WriteAllText(Path.Combine(pluginRoot, "plugin.json"), """
{
"name": "demo",
"version": "1.0.0",
"description": "Demo",
"agents": ["./linked/"]
}
""");
File.WriteAllText(Path.Combine(outsideDir, "outside.agent.md"), """
---
name: outside
description: External agent.
---
External.
""");
if (!SymlinkTestHelper.TryCreateDirectory(Path.Combine(pluginRoot, "linked"), outsideDir))
{
Directory.Delete(root, true);
return;
}
try
{
Assert.Empty(await AgentDiscovery.DiscoverAgentsInPlugin(pluginRoot));
}
finally
{
Directory.Delete(root, true);
}
}
[Fact]
public async Task PluginDiscoverySkipsLinkedAgentInsideConventionalDirectory()
{
var root = Path.Combine(Path.GetTempPath(), $"agent-mixed-link-{Guid.NewGuid():N}");
var pluginRoot = Path.Combine(root, "plugin");
var agentsDir = Path.Combine(pluginRoot, "agents");
var outsideDir = Path.Combine(root, "outside");
Directory.CreateDirectory(agentsDir);
Directory.CreateDirectory(outsideDir);
File.WriteAllText(Path.Combine(pluginRoot, "plugin.json"), """
{
"name": "demo",
"version": "1.0.0",
"description": "Demo",
"agents": ["./agents/"]
}
""");
File.WriteAllText(Path.Combine(agentsDir, "real.agent.md"), """
---
name: real
description: Real agent.
---
Real.
""");
var outsideAgent = Path.Combine(outsideDir, "linked.agent.md");
File.WriteAllText(outsideAgent, """
---
name: linked
description: External agent.
---
External.
""");
if (!SymlinkTestHelper.TryCreateFile(Path.Combine(agentsDir, "linked.agent.md"), outsideAgent))
{
Directory.Delete(root, true);
return;
}
try
{
var agent = Assert.Single(await AgentDiscovery.DiscoverAgentsInPlugin(pluginRoot));
Assert.Equal("real", agent.Name);
}
finally
{
Directory.Delete(root, true);
}
}
}
[Fact]
public void ValidAgentProducesNoErrors()
{
var content = "---\nname: test-agent\ndescription: A test agent.\n---\n# Test Agent\n\nDo the thing.\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, "test-agent", "A test agent."));
Assert.Empty(profile.Errors);
}
[Fact]
public void MissingFrontmatterErrors()
{
var content = "# Test Agent\n\nNo frontmatter here.\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content));
Assert.Contains(profile.Errors, e => e.Contains("frontmatter"));
}
[Fact]
public void MissingFrontmatterUsesFilenameAsProfileName()
{
var content = "# Test Agent\n\nNo frontmatter here.\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "", fileName: "my-agent.agent.md"));
Assert.Equal("my-agent.agent.md", profile.Name);
}
[Fact]
public void MissingNameErrors()
{
var content = "---\ndescription: A test agent.\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "", description: "A test agent."));
Assert.Contains(profile.Errors, e => e.Contains("name"));
}
[Fact]
public void MissingDescriptionErrors()
{
var content = "---\nname: test-agent\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, description: ""));
Assert.Contains(profile.Errors, e => e.Contains("description"));
}
[Fact]
public void DescriptionOverLimitErrors()
{
var desc = new string('a', 1025);
var content = $"---\nname: test-agent\ndescription: {desc}\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, description: desc));
Assert.Contains(profile.Errors, e => e.Contains("maximum"));
}
[Fact]
public void DescriptionAtLimitNoError()
{
var desc = new string('a', 1024);
var content = $"---\nname: test-agent\ndescription: {desc}\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, description: desc));
Assert.DoesNotContain(profile.Errors, e => e.Contains("maximum"));
}
[Fact]
public void NameNotMatchingFilenameErrors()
{
var content = "---\nname: my-agent\ndescription: test\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "my-agent", fileName: "different-agent.agent.md"));
Assert.Contains(profile.Errors, e => e.Contains("does not match filename"));
}
[Fact]
public void NameMatchingFilenameNoError()
{
var content = "---\nname: my-agent\ndescription: test\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "my-agent", fileName: "my-agent.agent.md"));
Assert.DoesNotContain(profile.Errors, e => e.Contains("does not match filename"));
}
[Fact]
public async Task DiscoveryPreservesDeclaredAgentDependencies()
{
var root = Path.Combine(Path.GetTempPath(), $"agent-discovery-{Guid.NewGuid():N}");
Directory.CreateDirectory(root);
try
{
File.WriteAllText(Path.Combine(root, "parent.agent.md"), """
---
name: parent
description: Parent agent.
agents:
- child-a
- child-b
---
# Parent
""");
var agent = Assert.Single(await AgentDiscovery.DiscoverAgentsInDirectory(root));
Assert.Equal(["child-a", "child-b"], agent.Agents);
}
finally
{
Directory.Delete(root, true);
}
}
[Fact]
public void NameWithUppercaseErrors()
{
var content = "---\nname: My-Agent\ndescription: test\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "My-Agent", fileName: "My-Agent.agent.md"));
Assert.Contains(profile.Errors, e => e.Contains("invalid characters"));
}
[Fact]
public void NameTooLongErrors()
{
var longName = new string('a', 65);
var content = $"---\nname: {longName}\ndescription: test\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: longName, fileName: $"{longName}.agent.md"));
Assert.Contains(profile.Errors, e => e.Contains("maximum is 64"));
}
[Fact]
public void NameStartingWithHyphenErrors()
{
var content = "---\nname: -my-agent\ndescription: test\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "-my-agent", fileName: "-my-agent.agent.md"));
Assert.Contains(profile.Errors, e => e.Contains("starts or ends with a hyphen"));
}
[Fact]
public void NameEndingWithHyphenErrors()
{
var content = "---\nname: my-agent-\ndescription: test\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "my-agent-", fileName: "my-agent-.agent.md"));
Assert.Contains(profile.Errors, e => e.Contains("starts or ends with a hyphen"));
}
[Fact]
public void NameWithConsecutiveHyphensErrors()
{
var content = "---\nname: my--agent\ndescription: test\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "my--agent", fileName: "my--agent.agent.md"));
Assert.Contains(profile.Errors, e => e.Contains("consecutive hyphens"));
}
[Fact]
public void ErrorMessagesSayAgentNotSkill()
{
var content = "---\nname: My-Agent\ndescription: test\n---\n# Test\n";
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "My-Agent", fileName: "My-Agent.agent.md"));
Assert.Contains(profile.Errors, e => e.StartsWith("Agent name"));
Assert.DoesNotContain(profile.Errors, e => e.StartsWith("Skill name"));
}
}
public class PluginProfilerTests
{
[Fact]
public void ValidPluginProducesNoErrors()
{
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(pluginDir);
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
Directory.CreateDirectory(Path.Combine(pluginDir, "agents"));
File.WriteAllText(Path.Combine(pluginDir, "agents", "test.agent.md"), "---\nname: test\ndescription: test\n---\n# Test\n");
var dirName = Path.GetFileName(pluginDir);
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], ["./agents/test.agent.md"], pluginDir, dirName);
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Empty(result.Errors);
Assert.Empty(result.Warnings);
}
finally
{
Directory.Delete(pluginDir, true);
}
}
[Fact]
public void MissingNameErrors()
{
var plugin = new PluginInfo("", "1.0.0", "desc", ["./skills/"], [], "/tmp/test", "test");
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("name"));
}
[Fact]
public void NameNotMatchingDirectoryErrors()
{
var plugin = new PluginInfo("wrong-name", "1.0.0", "desc", ["./skills/"], [], "/tmp/my-plugin", "my-plugin");
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("does not match directory"));
}
[Fact]
public void MissingVersionErrors()
{
var plugin = new PluginInfo("test", null, "desc", ["./skills/"], [], "/tmp/test", "test");
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("version"));
}
[Fact]
public void MissingDescriptionErrors()
{
var plugin = new PluginInfo("test", "1.0.0", null, ["./skills/"], [], "/tmp/test", "test");
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("description"));
}
[Fact]
public void DescriptionOverLimitErrors()
{
var desc = new string('a', 1025);
var plugin = new PluginInfo("test", "1.0.0", desc, ["./skills/"], [], "/tmp/test", "test");
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("maximum"));
}
[Fact]
public void MissingSkillsPathErrors()
{
var plugin = new PluginInfo("test", "1.0.0", "desc", [], [], "/tmp/test", "test");
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("skills"));
}
[Fact]
public void NonexistentSkillsPathErrors()
{
var plugin = new PluginInfo("test", "1.0.0", "desc", ["./nonexistent/"], [], "/tmp/test", "test");
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("does not exist"));
}
[Fact]
public void NonexistentAgentsPathErrors()
{
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(pluginDir);
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
var dirName = Path.GetFileName(pluginDir);
var plugin = new PluginInfo(dirName, "1.0.0", "desc", ["./skills/"], ["./nonexistent.agent.md"], pluginDir, dirName);
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("does not exist"));
}
finally
{
Directory.Delete(pluginDir, true);
}
}
[Fact]
public void NonexistentAgentFilePathErrors()
{
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(pluginDir);
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
var dirName = Path.GetFileName(pluginDir);
var plugin = new PluginInfo(dirName, "1.0.0", "desc", ["./skills/"], ["./agents/missing.agent.md"], pluginDir, dirName);
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("does not exist"));
}
finally
{
Directory.Delete(pluginDir, true);
}
}
[Fact]
public void AgentDirectoryPathErrors()
{
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(pluginDir);
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
Directory.CreateDirectory(Path.Combine(pluginDir, "agents"));
File.WriteAllText(Path.Combine(pluginDir, "agents", "test.agent.md"), "---\nname: test\ndescription: test\n---\n# Test\n");
var dirName = Path.GetFileName(pluginDir);
var plugin = new PluginInfo(dirName, "1.0.0", "desc", ["./skills/"], ["./agents/"], pluginDir, dirName);
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("is a directory") && e.Contains("explicit file paths"));
}
finally
{
Directory.Delete(pluginDir, true);
}
}
[Fact]
public void ValidAgentPathsArrayProducesNoWarnings()
{
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(pluginDir);
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
Directory.CreateDirectory(Path.Combine(pluginDir, "agents"));
File.WriteAllText(Path.Combine(pluginDir, "agents", "test.agent.md"), "---\nname: test\ndescription: test\n---\n# Test\n");
var dirName = Path.GetFileName(pluginDir);
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], ["./agents/test.agent.md"], pluginDir, dirName);
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Empty(result.Errors);
Assert.Empty(result.Warnings);
}
finally
{
Directory.Delete(pluginDir, true);
}
}
[Theory]
[InlineData("agents")]
[InlineData("lspServers")]
public void CodexManifestWithUnsupportedComponentErrors(string fieldName)
{
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
Directory.CreateDirectory(Path.Combine(pluginDir, ".codex-plugin"));
var dirName = Path.GetFileName(pluginDir);
File.WriteAllText(
Path.Combine(pluginDir, ".codex-plugin", "plugin.json"),
$$"""{"name":"{{dirName}}","version":"1.0.0","description":"A test plugin.","skills":["./skills/"],"{{fieldName}}":[]}""");
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], [], pluginDir, dirName);
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(
result.Errors,
e => e.Contains(".codex-plugin/plugin.json") &&
e.Contains($"unsupported Codex field '{fieldName}'"));
}
finally
{
Directory.Delete(pluginDir, true);
}
}
[Theory]
[InlineData("name", "[]", "field 'name' must be string")]
[InlineData("version", "{}", "field 'version' must be string")]
[InlineData("description", "[]", "field 'description' must be string")]
[InlineData("keywords", """["valid",1]""", "field 'keywords' must be an array of strings")]
[InlineData("skills", "{}", "field 'skills' must be a string or an array of strings")]
[InlineData("skills", "[]", "field 'skills' must contain at least one path")]
[InlineData("skills", """["skills"]""", "field 'skills' path 'skills' must start with './'")]
[InlineData("skills", """["./"]""", "field 'skills' path must not be './'")]
[InlineData("skills", """["./packs/../packs/"]""", "field 'skills' path './packs/../packs/' must not contain '..'")]
[InlineData("commands", "{}", "field 'commands' must be a string or an array of strings")]
[InlineData("apps", "[]", "field 'apps' must be string")]
[InlineData("hooks", "[true]", "field 'hooks' must be a string, object")]
[InlineData("hooks", """["./hooks.json",{"hooks":{}}]""", "homogeneous array of strings or objects")]
[InlineData("interface", "[]", "field 'interface' must be an object")]
public void CodexManifestWithInvalidFieldShapeErrors(string fieldName, string invalidJson, string expectedError)
{
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
Directory.CreateDirectory(Path.Combine(pluginDir, ".codex-plugin"));
var dirName = Path.GetFileName(pluginDir);
var properties = new Dictionary<string, string>
{
["name"] = JsonSerializer.Serialize(dirName),
["version"] = "\"1.0.0\"",
["description"] = "\"A test plugin.\"",
["skills"] = """["./skills/"]""",
};
properties[fieldName] = invalidJson;
var json = "{" + string.Join(",", properties.Select(p => $"\"{p.Key}\":{p.Value}")) + "}";
File.WriteAllText(Path.Combine(pluginDir, ".codex-plugin", "plugin.json"), json);
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], [], pluginDir, dirName);
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, error => error.Contains(expectedError));
}
finally
{
Directory.Delete(pluginDir, true);
}
}
[Theory]
[InlineData("name", "has no 'name' field")]
[InlineData("skills", "has no 'skills' field")]
public void CodexManifestMissingRequiredRepositoryFieldErrors(string omittedField, string expectedError)
{
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
Directory.CreateDirectory(Path.Combine(pluginDir, ".codex-plugin"));
var dirName = Path.GetFileName(pluginDir);
var properties = new Dictionary<string, string>
{
["name"] = JsonSerializer.Serialize(dirName),
["version"] = "\"1.0.0\"",
["description"] = "\"A test plugin.\"",
["skills"] = """["./skills/"]""",
};
properties.Remove(omittedField);
var json = "{" + string.Join(",", properties.Select(p => $"\"{p.Key}\":{p.Value}")) + "}";
File.WriteAllText(Path.Combine(pluginDir, ".codex-plugin", "plugin.json"), json);
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], [], pluginDir, dirName);
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, error => error.Contains(expectedError));
}
finally
{
Directory.Delete(pluginDir, true);
}
}
[Fact]
public void ValidSkillPathsArrayProducesNoErrors()
{
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(pluginDir);
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
var dirName = Path.GetFileName(pluginDir);
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], [], pluginDir, dirName);
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Empty(result.Errors);
}
finally
{
Directory.Delete(pluginDir, true);
}
}
[Fact]
public void NonexistentSkillPathInArrayErrors()
{
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(pluginDir);
var dirName = Path.GetFileName(pluginDir);
var plugin = new PluginInfo(dirName, "1.0.0", "desc", ["./nonexistent/"], [], pluginDir, dirName);
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("does not exist"));
}
finally
{
Directory.Delete(pluginDir, true);
}
}
[Fact]
public void NameFormatErrors()
{
var plugin = new PluginInfo("My_Plugin", "1.0.0", "desc", ["./skills/"], [], "/tmp/My_Plugin", "My_Plugin");
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("invalid characters"));
}
[Fact]
public void ErrorMessagesSayPluginNotSkill()
{
var plugin = new PluginInfo("My_Plugin", "1.0.0", "desc", ["./skills/"], [], "/tmp/My_Plugin", "My_Plugin");
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.StartsWith("Plugin name"));
Assert.DoesNotContain(result.Errors, e => e.StartsWith("Skill name"));
}
[Fact]
public void ParsePluginJsonReturnsNullForMissingFile()
{
var result = PluginDiscovery.ParsePluginJson("/nonexistent/plugin.json");
Assert.Null(result);
}
[Fact]
public void ParsePluginJsonParsesValidFile()
{
var dir = Path.Combine(Path.GetTempPath(), "parse-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(dir);
var jsonPath = Path.Combine(dir, "plugin.json");
File.WriteAllText(jsonPath, """{"name":"my-plugin","version":"0.1.0","description":"A plugin.","skills":["./skills/"],"agents":["./agents/"]}""");
var plugin = PluginDiscovery.ParsePluginJson(jsonPath);
Assert.NotNull(plugin);
Assert.Equal("my-plugin", plugin.Name);
Assert.Equal("0.1.0", plugin.Version);
Assert.Equal("A plugin.", plugin.Description);
Assert.Single(plugin.SkillPaths);
Assert.Equal("./skills/", plugin.SkillPaths[0]);
Assert.Single(plugin.AgentPaths);
Assert.Equal("./agents/", plugin.AgentPaths[0]);
}
finally
{
Directory.Delete(dir, true);
}
}
[Fact]
public void ParsePluginJsonNormalizesStringToArray()
{
var dir = Path.Combine(Path.GetTempPath(), "parse-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(dir);
var jsonPath = Path.Combine(dir, "plugin.json");
File.WriteAllText(jsonPath, """{"name":"my-plugin","version":"0.1.0","description":"A plugin.","skills":"./skills/","agents":"./agents/"}""");
var plugin = PluginDiscovery.ParsePluginJson(jsonPath);
Assert.NotNull(plugin);
Assert.Single(plugin.SkillPaths);
Assert.Equal("./skills/", plugin.SkillPaths[0]);
Assert.Single(plugin.AgentPaths);
Assert.Equal("./agents/", plugin.AgentPaths[0]);
}
finally
{
Directory.Delete(dir, true);
}
}
[Fact]
public void ParsePluginJsonNoAgentsField()
{
var dir = Path.Combine(Path.GetTempPath(), "parse-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(dir);
var jsonPath = Path.Combine(dir, "plugin.json");
File.WriteAllText(jsonPath, """{"name":"my-plugin","version":"0.1.0","description":"A plugin.","skills":"./skills/"}""");
var plugin = PluginDiscovery.ParsePluginJson(jsonPath);
Assert.NotNull(plugin);
Assert.Empty(plugin.AgentPaths);
}
finally
{
Directory.Delete(dir, true);
}
}
[Fact]
public void ParsePluginJsonThrowsOnMalformedJson()
{
var dir = Path.Combine(Path.GetTempPath(), "parse-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(dir);
var jsonPath = Path.Combine(dir, "plugin.json");
File.WriteAllText(jsonPath, "{ not valid json!!!");
Assert.Throws<JsonException>(() => PluginDiscovery.ParsePluginJson(jsonPath));
}
finally
{
Directory.Delete(dir, true);
}
}
// A non-object root would make TryGetProperty throw InvalidOperationException, which callers
// catching JsonException would not handle.
[Theory]
[InlineData("[]", "array")]
[InlineData("null", "null")]
[InlineData("\"a string\"", "string")]
[InlineData("42", "number")]
public void ParsePluginJsonThrowsJsonExceptionOnNonObjectRoot(string json, string expectedKind)
{
var dir = Path.Combine(Path.GetTempPath(), "parse-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(dir);
var jsonPath = Path.Combine(dir, "plugin.json");
File.WriteAllText(jsonPath, json);
var ex = Assert.Throws<JsonException>(() => PluginDiscovery.ParsePluginJson(jsonPath));
Assert.Contains($"root value is {expectedKind}", ex.Message);
}
finally
{
Directory.Delete(dir, true);
}
}
[Fact]
public void AbsoluteSkillsPathErrors()
{
var plugin = new PluginInfo("test", "1.0.0", "desc", ["/etc/skills/"], [], "/tmp/test", "test");
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("invalid") && e.Contains("absolute"));
}
[Fact]
public void TraversalSkillsPathErrors()
{
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
try
{
Directory.CreateDirectory(pluginDir);
var dirName = Path.GetFileName(pluginDir);
var plugin = new PluginInfo(dirName, "1.0.0", "desc", ["../../../etc/"], [], pluginDir, dirName);
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Contains(result.Errors, e => e.Contains("invalid") && e.Contains("outside"));
}
finally
{
Directory.Delete(pluginDir, true);
}
}
[Fact]
public void MissingNameFallsBackToDirectoryName()
{
var plugin = new PluginInfo("", "1.0.0", "desc", ["./skills/"], [], "/tmp/my-plugin", "my-plugin");
var result = PluginProfiler.ValidatePlugin(plugin);
Assert.Equal("my-plugin", result.Name);
}
}