mirror of
https://github.com/dotnet/skills.git
synced 2026-09-20 09:49:54 +08:00
36222bf32d
* feat(evaluation): add custom agent coverage Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): address agent review feedback Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): reject linked fixture sources Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): preserve agent result invariants Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): fail closed on agent errors Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): preserve completion regressions Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): preserve nested command quotes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): harden native agent evidence Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): honor declared agent layout Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): resolve declared agent sources Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): secure agent path discovery Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): reject linked dependencies Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): centralize path safety checks Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): diagnose ambiguous dependencies Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): reject linked allowed roots Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): preserve skill agent isolation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): normalize dashboard evidence Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): preserve agent gate semantics Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): fail closed on incomplete evidence Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): preserve completion evidence Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): reject overflowing durations Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): stage verified plugin skills Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): block shell network access Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): reject linked MCP config files Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): trust manual dispatch path safety Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): keep agent plugin activation diagnostic Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): count failed tool completions Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix(evaluation): synchronize agent event capture Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
784 lines
30 KiB
C#
784 lines
30 KiB
C#
using System.Text.Json;
|
|
using SkillValidator.Check;
|
|
using SkillValidator.Shared;
|
|
|
|
namespace SkillValidator.Tests;
|
|
|
|
public class AgentProfilerTests
|
|
{
|
|
private static AgentInfo MakeAgent(
|
|
string content,
|
|
string name = "test-agent",
|
|
string description = "Test agent description",
|
|
string fileName = "test-agent.agent.md")
|
|
{
|
|
return new AgentInfo(name, description, $"/tmp/agents/{fileName}", content, fileName);
|
|
}
|
|
|
|
public class AgentDiscoveryPathSafetyTests
|
|
{
|
|
[Fact]
|
|
public async Task PluginDiscoveryRejectsDeclaredAgentFileSymlink()
|
|
{
|
|
var root = Path.Combine(Path.GetTempPath(), $"agent-file-link-{Guid.NewGuid():N}");
|
|
var pluginRoot = Path.Combine(root, "plugin");
|
|
var agentsDir = Path.Combine(pluginRoot, "agents");
|
|
var outsideDir = Path.Combine(root, "outside");
|
|
Directory.CreateDirectory(agentsDir);
|
|
Directory.CreateDirectory(outsideDir);
|
|
File.WriteAllText(Path.Combine(pluginRoot, "plugin.json"), """
|
|
{
|
|
"name": "demo",
|
|
"version": "1.0.0",
|
|
"description": "Demo",
|
|
"agents": ["./agents/leak.agent.md"]
|
|
}
|
|
""");
|
|
var outsideAgent = Path.Combine(outsideDir, "leak.agent.md");
|
|
File.WriteAllText(outsideAgent, """
|
|
---
|
|
name: leak
|
|
description: External agent.
|
|
---
|
|
External.
|
|
""");
|
|
if (!SymlinkTestHelper.TryCreateFile(Path.Combine(agentsDir, "leak.agent.md"), outsideAgent))
|
|
{
|
|
Directory.Delete(root, true);
|
|
return;
|
|
}
|
|
try
|
|
{
|
|
Assert.Empty(await AgentDiscovery.DiscoverAgentsInPlugin(pluginRoot));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(root, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PluginDiscoveryRejectsDeclaredDirectorySymlink()
|
|
{
|
|
var root = Path.Combine(Path.GetTempPath(), $"agent-dir-link-{Guid.NewGuid():N}");
|
|
var pluginRoot = Path.Combine(root, "plugin");
|
|
var outsideDir = Path.Combine(root, "outside");
|
|
Directory.CreateDirectory(pluginRoot);
|
|
Directory.CreateDirectory(outsideDir);
|
|
File.WriteAllText(Path.Combine(pluginRoot, "plugin.json"), """
|
|
{
|
|
"name": "demo",
|
|
"version": "1.0.0",
|
|
"description": "Demo",
|
|
"agents": ["./linked/"]
|
|
}
|
|
""");
|
|
File.WriteAllText(Path.Combine(outsideDir, "outside.agent.md"), """
|
|
---
|
|
name: outside
|
|
description: External agent.
|
|
---
|
|
External.
|
|
""");
|
|
if (!SymlinkTestHelper.TryCreateDirectory(Path.Combine(pluginRoot, "linked"), outsideDir))
|
|
{
|
|
Directory.Delete(root, true);
|
|
return;
|
|
}
|
|
try
|
|
{
|
|
Assert.Empty(await AgentDiscovery.DiscoverAgentsInPlugin(pluginRoot));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(root, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PluginDiscoverySkipsLinkedAgentInsideConventionalDirectory()
|
|
{
|
|
var root = Path.Combine(Path.GetTempPath(), $"agent-mixed-link-{Guid.NewGuid():N}");
|
|
var pluginRoot = Path.Combine(root, "plugin");
|
|
var agentsDir = Path.Combine(pluginRoot, "agents");
|
|
var outsideDir = Path.Combine(root, "outside");
|
|
Directory.CreateDirectory(agentsDir);
|
|
Directory.CreateDirectory(outsideDir);
|
|
File.WriteAllText(Path.Combine(pluginRoot, "plugin.json"), """
|
|
{
|
|
"name": "demo",
|
|
"version": "1.0.0",
|
|
"description": "Demo",
|
|
"agents": ["./agents/"]
|
|
}
|
|
""");
|
|
File.WriteAllText(Path.Combine(agentsDir, "real.agent.md"), """
|
|
---
|
|
name: real
|
|
description: Real agent.
|
|
---
|
|
Real.
|
|
""");
|
|
var outsideAgent = Path.Combine(outsideDir, "linked.agent.md");
|
|
File.WriteAllText(outsideAgent, """
|
|
---
|
|
name: linked
|
|
description: External agent.
|
|
---
|
|
External.
|
|
""");
|
|
if (!SymlinkTestHelper.TryCreateFile(Path.Combine(agentsDir, "linked.agent.md"), outsideAgent))
|
|
{
|
|
Directory.Delete(root, true);
|
|
return;
|
|
}
|
|
try
|
|
{
|
|
var agent = Assert.Single(await AgentDiscovery.DiscoverAgentsInPlugin(pluginRoot));
|
|
Assert.Equal("real", agent.Name);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(root, true);
|
|
}
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ValidAgentProducesNoErrors()
|
|
{
|
|
var content = "---\nname: test-agent\ndescription: A test agent.\n---\n# Test Agent\n\nDo the thing.\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, "test-agent", "A test agent."));
|
|
Assert.Empty(profile.Errors);
|
|
}
|
|
|
|
[Fact]
|
|
public void MissingFrontmatterErrors()
|
|
{
|
|
var content = "# Test Agent\n\nNo frontmatter here.\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content));
|
|
Assert.Contains(profile.Errors, e => e.Contains("frontmatter"));
|
|
}
|
|
|
|
[Fact]
|
|
public void MissingFrontmatterUsesFilenameAsProfileName()
|
|
{
|
|
var content = "# Test Agent\n\nNo frontmatter here.\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "", fileName: "my-agent.agent.md"));
|
|
Assert.Equal("my-agent.agent.md", profile.Name);
|
|
}
|
|
|
|
[Fact]
|
|
public void MissingNameErrors()
|
|
{
|
|
var content = "---\ndescription: A test agent.\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "", description: "A test agent."));
|
|
Assert.Contains(profile.Errors, e => e.Contains("name"));
|
|
}
|
|
|
|
[Fact]
|
|
public void MissingDescriptionErrors()
|
|
{
|
|
var content = "---\nname: test-agent\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, description: ""));
|
|
Assert.Contains(profile.Errors, e => e.Contains("description"));
|
|
}
|
|
|
|
[Fact]
|
|
public void DescriptionOverLimitErrors()
|
|
{
|
|
var desc = new string('a', 1025);
|
|
var content = $"---\nname: test-agent\ndescription: {desc}\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, description: desc));
|
|
Assert.Contains(profile.Errors, e => e.Contains("maximum"));
|
|
}
|
|
|
|
[Fact]
|
|
public void DescriptionAtLimitNoError()
|
|
{
|
|
var desc = new string('a', 1024);
|
|
var content = $"---\nname: test-agent\ndescription: {desc}\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, description: desc));
|
|
Assert.DoesNotContain(profile.Errors, e => e.Contains("maximum"));
|
|
}
|
|
|
|
[Fact]
|
|
public void NameNotMatchingFilenameErrors()
|
|
{
|
|
var content = "---\nname: my-agent\ndescription: test\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "my-agent", fileName: "different-agent.agent.md"));
|
|
Assert.Contains(profile.Errors, e => e.Contains("does not match filename"));
|
|
}
|
|
|
|
[Fact]
|
|
public void NameMatchingFilenameNoError()
|
|
{
|
|
var content = "---\nname: my-agent\ndescription: test\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "my-agent", fileName: "my-agent.agent.md"));
|
|
Assert.DoesNotContain(profile.Errors, e => e.Contains("does not match filename"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DiscoveryPreservesDeclaredAgentDependencies()
|
|
{
|
|
var root = Path.Combine(Path.GetTempPath(), $"agent-discovery-{Guid.NewGuid():N}");
|
|
Directory.CreateDirectory(root);
|
|
try
|
|
{
|
|
File.WriteAllText(Path.Combine(root, "parent.agent.md"), """
|
|
---
|
|
name: parent
|
|
description: Parent agent.
|
|
agents:
|
|
- child-a
|
|
- child-b
|
|
---
|
|
# Parent
|
|
""");
|
|
|
|
var agent = Assert.Single(await AgentDiscovery.DiscoverAgentsInDirectory(root));
|
|
|
|
Assert.Equal(["child-a", "child-b"], agent.Agents);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(root, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void NameWithUppercaseErrors()
|
|
{
|
|
var content = "---\nname: My-Agent\ndescription: test\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "My-Agent", fileName: "My-Agent.agent.md"));
|
|
Assert.Contains(profile.Errors, e => e.Contains("invalid characters"));
|
|
}
|
|
|
|
[Fact]
|
|
public void NameTooLongErrors()
|
|
{
|
|
var longName = new string('a', 65);
|
|
var content = $"---\nname: {longName}\ndescription: test\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: longName, fileName: $"{longName}.agent.md"));
|
|
Assert.Contains(profile.Errors, e => e.Contains("maximum is 64"));
|
|
}
|
|
|
|
[Fact]
|
|
public void NameStartingWithHyphenErrors()
|
|
{
|
|
var content = "---\nname: -my-agent\ndescription: test\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "-my-agent", fileName: "-my-agent.agent.md"));
|
|
Assert.Contains(profile.Errors, e => e.Contains("starts or ends with a hyphen"));
|
|
}
|
|
|
|
[Fact]
|
|
public void NameEndingWithHyphenErrors()
|
|
{
|
|
var content = "---\nname: my-agent-\ndescription: test\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "my-agent-", fileName: "my-agent-.agent.md"));
|
|
Assert.Contains(profile.Errors, e => e.Contains("starts or ends with a hyphen"));
|
|
}
|
|
|
|
[Fact]
|
|
public void NameWithConsecutiveHyphensErrors()
|
|
{
|
|
var content = "---\nname: my--agent\ndescription: test\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "my--agent", fileName: "my--agent.agent.md"));
|
|
Assert.Contains(profile.Errors, e => e.Contains("consecutive hyphens"));
|
|
}
|
|
|
|
[Fact]
|
|
public void ErrorMessagesSayAgentNotSkill()
|
|
{
|
|
var content = "---\nname: My-Agent\ndescription: test\n---\n# Test\n";
|
|
var profile = AgentProfiler.AnalyzeAgent(MakeAgent(content, name: "My-Agent", fileName: "My-Agent.agent.md"));
|
|
Assert.Contains(profile.Errors, e => e.StartsWith("Agent name"));
|
|
Assert.DoesNotContain(profile.Errors, e => e.StartsWith("Skill name"));
|
|
}
|
|
}
|
|
|
|
public class PluginProfilerTests
|
|
{
|
|
[Fact]
|
|
public void ValidPluginProducesNoErrors()
|
|
{
|
|
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(pluginDir);
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "agents"));
|
|
File.WriteAllText(Path.Combine(pluginDir, "agents", "test.agent.md"), "---\nname: test\ndescription: test\n---\n# Test\n");
|
|
var dirName = Path.GetFileName(pluginDir);
|
|
|
|
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], ["./agents/test.agent.md"], pluginDir, dirName);
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Empty(result.Errors);
|
|
Assert.Empty(result.Warnings);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(pluginDir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void MissingNameErrors()
|
|
{
|
|
var plugin = new PluginInfo("", "1.0.0", "desc", ["./skills/"], [], "/tmp/test", "test");
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("name"));
|
|
}
|
|
|
|
[Fact]
|
|
public void NameNotMatchingDirectoryErrors()
|
|
{
|
|
var plugin = new PluginInfo("wrong-name", "1.0.0", "desc", ["./skills/"], [], "/tmp/my-plugin", "my-plugin");
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("does not match directory"));
|
|
}
|
|
|
|
[Fact]
|
|
public void MissingVersionErrors()
|
|
{
|
|
var plugin = new PluginInfo("test", null, "desc", ["./skills/"], [], "/tmp/test", "test");
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("version"));
|
|
}
|
|
|
|
[Fact]
|
|
public void MissingDescriptionErrors()
|
|
{
|
|
var plugin = new PluginInfo("test", "1.0.0", null, ["./skills/"], [], "/tmp/test", "test");
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("description"));
|
|
}
|
|
|
|
[Fact]
|
|
public void DescriptionOverLimitErrors()
|
|
{
|
|
var desc = new string('a', 1025);
|
|
var plugin = new PluginInfo("test", "1.0.0", desc, ["./skills/"], [], "/tmp/test", "test");
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("maximum"));
|
|
}
|
|
|
|
[Fact]
|
|
public void MissingSkillsPathErrors()
|
|
{
|
|
var plugin = new PluginInfo("test", "1.0.0", "desc", [], [], "/tmp/test", "test");
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("skills"));
|
|
}
|
|
|
|
[Fact]
|
|
public void NonexistentSkillsPathErrors()
|
|
{
|
|
var plugin = new PluginInfo("test", "1.0.0", "desc", ["./nonexistent/"], [], "/tmp/test", "test");
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("does not exist"));
|
|
}
|
|
|
|
[Fact]
|
|
public void NonexistentAgentsPathErrors()
|
|
{
|
|
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(pluginDir);
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
|
|
var dirName = Path.GetFileName(pluginDir);
|
|
|
|
var plugin = new PluginInfo(dirName, "1.0.0", "desc", ["./skills/"], ["./nonexistent.agent.md"], pluginDir, dirName);
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("does not exist"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(pluginDir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void NonexistentAgentFilePathErrors()
|
|
{
|
|
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(pluginDir);
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
|
|
var dirName = Path.GetFileName(pluginDir);
|
|
|
|
var plugin = new PluginInfo(dirName, "1.0.0", "desc", ["./skills/"], ["./agents/missing.agent.md"], pluginDir, dirName);
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("does not exist"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(pluginDir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void AgentDirectoryPathErrors()
|
|
{
|
|
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(pluginDir);
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "agents"));
|
|
File.WriteAllText(Path.Combine(pluginDir, "agents", "test.agent.md"), "---\nname: test\ndescription: test\n---\n# Test\n");
|
|
var dirName = Path.GetFileName(pluginDir);
|
|
|
|
var plugin = new PluginInfo(dirName, "1.0.0", "desc", ["./skills/"], ["./agents/"], pluginDir, dirName);
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("is a directory") && e.Contains("explicit file paths"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(pluginDir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ValidAgentPathsArrayProducesNoWarnings()
|
|
{
|
|
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(pluginDir);
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "agents"));
|
|
File.WriteAllText(Path.Combine(pluginDir, "agents", "test.agent.md"), "---\nname: test\ndescription: test\n---\n# Test\n");
|
|
var dirName = Path.GetFileName(pluginDir);
|
|
|
|
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], ["./agents/test.agent.md"], pluginDir, dirName);
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Empty(result.Errors);
|
|
Assert.Empty(result.Warnings);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(pluginDir, true);
|
|
}
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("agents")]
|
|
[InlineData("lspServers")]
|
|
public void CodexManifestWithUnsupportedComponentErrors(string fieldName)
|
|
{
|
|
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, ".codex-plugin"));
|
|
var dirName = Path.GetFileName(pluginDir);
|
|
|
|
File.WriteAllText(
|
|
Path.Combine(pluginDir, ".codex-plugin", "plugin.json"),
|
|
$$"""{"name":"{{dirName}}","version":"1.0.0","description":"A test plugin.","skills":["./skills/"],"{{fieldName}}":[]}""");
|
|
|
|
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], [], pluginDir, dirName);
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
|
|
Assert.Contains(
|
|
result.Errors,
|
|
e => e.Contains(".codex-plugin/plugin.json") &&
|
|
e.Contains($"unsupported Codex field '{fieldName}'"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(pluginDir, true);
|
|
}
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("name", "[]", "field 'name' must be string")]
|
|
[InlineData("version", "{}", "field 'version' must be string")]
|
|
[InlineData("description", "[]", "field 'description' must be string")]
|
|
[InlineData("keywords", """["valid",1]""", "field 'keywords' must be an array of strings")]
|
|
[InlineData("skills", "{}", "field 'skills' must be a string or an array of strings")]
|
|
[InlineData("skills", "[]", "field 'skills' must contain at least one path")]
|
|
[InlineData("skills", """["skills"]""", "field 'skills' path 'skills' must start with './'")]
|
|
[InlineData("skills", """["./"]""", "field 'skills' path must not be './'")]
|
|
[InlineData("skills", """["./packs/../packs/"]""", "field 'skills' path './packs/../packs/' must not contain '..'")]
|
|
[InlineData("commands", "{}", "field 'commands' must be a string or an array of strings")]
|
|
[InlineData("apps", "[]", "field 'apps' must be string")]
|
|
[InlineData("hooks", "[true]", "field 'hooks' must be a string, object")]
|
|
[InlineData("hooks", """["./hooks.json",{"hooks":{}}]""", "homogeneous array of strings or objects")]
|
|
[InlineData("interface", "[]", "field 'interface' must be an object")]
|
|
public void CodexManifestWithInvalidFieldShapeErrors(string fieldName, string invalidJson, string expectedError)
|
|
{
|
|
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, ".codex-plugin"));
|
|
var dirName = Path.GetFileName(pluginDir);
|
|
var properties = new Dictionary<string, string>
|
|
{
|
|
["name"] = JsonSerializer.Serialize(dirName),
|
|
["version"] = "\"1.0.0\"",
|
|
["description"] = "\"A test plugin.\"",
|
|
["skills"] = """["./skills/"]""",
|
|
};
|
|
properties[fieldName] = invalidJson;
|
|
var json = "{" + string.Join(",", properties.Select(p => $"\"{p.Key}\":{p.Value}")) + "}";
|
|
File.WriteAllText(Path.Combine(pluginDir, ".codex-plugin", "plugin.json"), json);
|
|
|
|
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], [], pluginDir, dirName);
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
|
|
Assert.Contains(result.Errors, error => error.Contains(expectedError));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(pluginDir, true);
|
|
}
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("name", "has no 'name' field")]
|
|
[InlineData("skills", "has no 'skills' field")]
|
|
public void CodexManifestMissingRequiredRepositoryFieldErrors(string omittedField, string expectedError)
|
|
{
|
|
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, ".codex-plugin"));
|
|
var dirName = Path.GetFileName(pluginDir);
|
|
var properties = new Dictionary<string, string>
|
|
{
|
|
["name"] = JsonSerializer.Serialize(dirName),
|
|
["version"] = "\"1.0.0\"",
|
|
["description"] = "\"A test plugin.\"",
|
|
["skills"] = """["./skills/"]""",
|
|
};
|
|
properties.Remove(omittedField);
|
|
var json = "{" + string.Join(",", properties.Select(p => $"\"{p.Key}\":{p.Value}")) + "}";
|
|
File.WriteAllText(Path.Combine(pluginDir, ".codex-plugin", "plugin.json"), json);
|
|
|
|
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], [], pluginDir, dirName);
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
|
|
Assert.Contains(result.Errors, error => error.Contains(expectedError));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(pluginDir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ValidSkillPathsArrayProducesNoErrors()
|
|
{
|
|
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(pluginDir);
|
|
Directory.CreateDirectory(Path.Combine(pluginDir, "skills"));
|
|
var dirName = Path.GetFileName(pluginDir);
|
|
|
|
var plugin = new PluginInfo(dirName, "1.0.0", "A test plugin.", ["./skills/"], [], pluginDir, dirName);
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Empty(result.Errors);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(pluginDir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void NonexistentSkillPathInArrayErrors()
|
|
{
|
|
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(pluginDir);
|
|
var dirName = Path.GetFileName(pluginDir);
|
|
|
|
var plugin = new PluginInfo(dirName, "1.0.0", "desc", ["./nonexistent/"], [], pluginDir, dirName);
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("does not exist"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(pluginDir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void NameFormatErrors()
|
|
{
|
|
var plugin = new PluginInfo("My_Plugin", "1.0.0", "desc", ["./skills/"], [], "/tmp/My_Plugin", "My_Plugin");
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("invalid characters"));
|
|
}
|
|
|
|
[Fact]
|
|
public void ErrorMessagesSayPluginNotSkill()
|
|
{
|
|
var plugin = new PluginInfo("My_Plugin", "1.0.0", "desc", ["./skills/"], [], "/tmp/My_Plugin", "My_Plugin");
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.StartsWith("Plugin name"));
|
|
Assert.DoesNotContain(result.Errors, e => e.StartsWith("Skill name"));
|
|
}
|
|
|
|
[Fact]
|
|
public void ParsePluginJsonReturnsNullForMissingFile()
|
|
{
|
|
var result = PluginDiscovery.ParsePluginJson("/nonexistent/plugin.json");
|
|
Assert.Null(result);
|
|
}
|
|
|
|
[Fact]
|
|
public void ParsePluginJsonParsesValidFile()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "parse-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(dir);
|
|
var jsonPath = Path.Combine(dir, "plugin.json");
|
|
File.WriteAllText(jsonPath, """{"name":"my-plugin","version":"0.1.0","description":"A plugin.","skills":["./skills/"],"agents":["./agents/"]}""");
|
|
|
|
var plugin = PluginDiscovery.ParsePluginJson(jsonPath);
|
|
Assert.NotNull(plugin);
|
|
Assert.Equal("my-plugin", plugin.Name);
|
|
Assert.Equal("0.1.0", plugin.Version);
|
|
Assert.Equal("A plugin.", plugin.Description);
|
|
Assert.Single(plugin.SkillPaths);
|
|
Assert.Equal("./skills/", plugin.SkillPaths[0]);
|
|
Assert.Single(plugin.AgentPaths);
|
|
Assert.Equal("./agents/", plugin.AgentPaths[0]);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(dir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ParsePluginJsonNormalizesStringToArray()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "parse-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(dir);
|
|
var jsonPath = Path.Combine(dir, "plugin.json");
|
|
File.WriteAllText(jsonPath, """{"name":"my-plugin","version":"0.1.0","description":"A plugin.","skills":"./skills/","agents":"./agents/"}""");
|
|
|
|
var plugin = PluginDiscovery.ParsePluginJson(jsonPath);
|
|
Assert.NotNull(plugin);
|
|
Assert.Single(plugin.SkillPaths);
|
|
Assert.Equal("./skills/", plugin.SkillPaths[0]);
|
|
Assert.Single(plugin.AgentPaths);
|
|
Assert.Equal("./agents/", plugin.AgentPaths[0]);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(dir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ParsePluginJsonNoAgentsField()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "parse-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(dir);
|
|
var jsonPath = Path.Combine(dir, "plugin.json");
|
|
File.WriteAllText(jsonPath, """{"name":"my-plugin","version":"0.1.0","description":"A plugin.","skills":"./skills/"}""");
|
|
|
|
var plugin = PluginDiscovery.ParsePluginJson(jsonPath);
|
|
Assert.NotNull(plugin);
|
|
Assert.Empty(plugin.AgentPaths);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(dir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ParsePluginJsonThrowsOnMalformedJson()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "parse-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(dir);
|
|
var jsonPath = Path.Combine(dir, "plugin.json");
|
|
File.WriteAllText(jsonPath, "{ not valid json!!!");
|
|
|
|
Assert.Throws<JsonException>(() => PluginDiscovery.ParsePluginJson(jsonPath));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(dir, true);
|
|
}
|
|
}
|
|
|
|
// A non-object root would make TryGetProperty throw InvalidOperationException, which callers
|
|
// catching JsonException would not handle.
|
|
[Theory]
|
|
[InlineData("[]", "array")]
|
|
[InlineData("null", "null")]
|
|
[InlineData("\"a string\"", "string")]
|
|
[InlineData("42", "number")]
|
|
public void ParsePluginJsonThrowsJsonExceptionOnNonObjectRoot(string json, string expectedKind)
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "parse-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(dir);
|
|
var jsonPath = Path.Combine(dir, "plugin.json");
|
|
File.WriteAllText(jsonPath, json);
|
|
|
|
var ex = Assert.Throws<JsonException>(() => PluginDiscovery.ParsePluginJson(jsonPath));
|
|
Assert.Contains($"root value is {expectedKind}", ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(dir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void AbsoluteSkillsPathErrors()
|
|
{
|
|
var plugin = new PluginInfo("test", "1.0.0", "desc", ["/etc/skills/"], [], "/tmp/test", "test");
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("invalid") && e.Contains("absolute"));
|
|
}
|
|
|
|
[Fact]
|
|
public void TraversalSkillsPathErrors()
|
|
{
|
|
var pluginDir = Path.Combine(Path.GetTempPath(), "plugin-test-" + Guid.NewGuid().ToString("N"));
|
|
try
|
|
{
|
|
Directory.CreateDirectory(pluginDir);
|
|
var dirName = Path.GetFileName(pluginDir);
|
|
var plugin = new PluginInfo(dirName, "1.0.0", "desc", ["../../../etc/"], [], pluginDir, dirName);
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Contains(result.Errors, e => e.Contains("invalid") && e.Contains("outside"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(pluginDir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void MissingNameFallsBackToDirectoryName()
|
|
{
|
|
var plugin = new PluginInfo("", "1.0.0", "desc", ["./skills/"], [], "/tmp/my-plugin", "my-plugin");
|
|
var result = PluginProfiler.ValidatePlugin(plugin);
|
|
Assert.Equal("my-plugin", result.Name);
|
|
}
|
|
}
|