skills/dotnet__skills
1
0
Fork 0
mirror of https://github.com/dotnet/skills.git synced 2026-09-20 09:49:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Reintroduce copilot sessions storing (#530)

Browse Source
This commit is contained in:
Jan Krivanek
2026-04-17 14:58:15 +02:00
committed by GitHub
parent e294085cf0
commit 05e2868b00
6 changed files with 271 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/evaluation.yml
+1 -1
View File
@@ -1197,7 +1197,7 @@ jobs:
!cancelled() &&
needs.discover.result == 'success' &&
needs.discover.outputs.has_plugins == 'true' &&
github.ref == 'refs/heads/main'
(github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main'))
concurrency:
group: deploy-dashboard
cancel-in-progress: false
eng/dashboard/build-replay-sessions.ps1
+34 -5
View File
@@ -143,6 +143,13 @@ $manifestSessions = @()
$artifactDirs = Get-ChildItem -Path $ResultsDir -Directory -ErrorAction SilentlyContinue |
Where-Object { $_.Name -like 'skill-validator-results-*' }
Write-Host "Scanning $ResultsDir for session artifacts..."
$allDirs = @(Get-ChildItem -Path $ResultsDir -Directory -ErrorAction SilentlyContinue)
Write-Host " Total directories in ResultsDir: $($allDirs.Count)"
if ($allDirs.Count -gt 0) {
Write-Host " Directory names: $($allDirs.Name -join ', ')"
}
if (-not $artifactDirs) {
Write-Warning "No skill-validator-results-* directories found in $ResultsDir"
# Write empty manifest
@@ -159,23 +166,30 @@ foreach ($artifactDir in $artifactDirs) {
$pluginName = ($entryName -split '--')[0]
# Find timestamped result directory
$runDir = Get-ChildItem -Path $artifactDir.FullName -Directory -ErrorAction SilentlyContinue |
$allSubDirs = @(Get-ChildItem -Path $artifactDir.FullName -Directory -ErrorAction SilentlyContinue)
$runDir = $allSubDirs |
Where-Object { $_.Name -match '^\d{8}-\d{6}$' } |
Sort-Object Name -Descending |
Select-Object -First 1
if (-not $runDir) {
Write-Warning "No timestamped run directory found in $($artifactDir.Name), skipping"
Write-Warning "No timestamped run directory found in $($artifactDir.Name), skipping (subdirs: $($allSubDirs.Name -join ', '))"
continue
}
$sessionsDbPath = Join-Path $runDir.FullName "sessions.db"
if (-not (Test-Path $sessionsDbPath)) {
Write-Warning "No sessions.db found in $($runDir.FullName), skipping"
# List files in the run directory for diagnostics
$runFiles = Get-ChildItem -Path $runDir.FullName -File -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Name
Write-Warning "No sessions.db found in $($runDir.FullName), skipping (files: $($runFiles -join ', '))"
continue
}
Write-Host "Processing sessions from $($artifactDir.Name) ($($runDir.Name))..."
# Check for WAL file alongside the database
$walPath = "$sessionsDbPath-wal"
$hasWal = Test-Path $walPath
$dbSize = (Get-Item $sessionsDbPath).Length
Write-Host "Processing sessions from $($artifactDir.Name) ($($runDir.Name))... [db=$([math]::Round($dbSize/1024,1))KB, wal=$hasWal]"
# Query sessions.db
$query = "SELECT id, skill_name, scenario_name, role, run_index, model, status, config_dir FROM sessions WHERE status IN ('completed', 'timed_out') ORDER BY skill_name, scenario_name, run_index, role;"
@@ -187,6 +201,17 @@ foreach ($artifactDir in $artifactDirs) {
continue
}
Write-Host " Found $($rows.Count) completed/timed_out session(s) in DB"
if ($rows.Count -eq 0) {
# Query total sessions (any status) for diagnostics
try {
$allRows = Invoke-SqliteQuery -DatabasePath $sessionsDbPath -Query "SELECT id, status FROM sessions;"
Write-Warning " DB has $($allRows.Count) total session(s) with statuses: $(($allRows | ForEach-Object { ($_ -split '\|')[1] } | Sort-Object -Unique) -join ', ')"
} catch {
Write-Warning " Could not query all sessions: $_"
}
}
foreach ($row in $rows) {
if (-not $row) { continue }
$fields = $row -split '\|'
@@ -234,7 +259,7 @@ foreach ($artifactDir in $artifactDirs) {
$displayName = "$pluginName / $scenarioName ($roleTag, run $runIndex)"
$id = "$subDir/$pluginName/$safeScenario--$roleTag--run$runIndex"
$tags = @($Source, $pluginName, $roleTag)
$tags = @($Source, $pluginName, $roleTag, $safeScenario)
if ($Source -eq 'pr' -and $PrNumber -gt 0) {
$tags += "pr-$PrNumber"
}
@@ -264,3 +289,7 @@ $manifestPath = Join-Path $OutputDir "manifest.json"
$manifest | ConvertTo-Json -Depth 10 | Out-File -FilePath $manifestPath -Encoding utf8
Write-Host "`nManifest written to $manifestPath with $($manifestSessions.Count) session(s)"
if ($manifestSessions.Count -eq 0) {
Write-Warning "No sessions were extracted from any artifact. Check warnings above for details."
Write-Warning "Artifact dirs scanned: $($artifactDirs.Count)"
}
eng/dashboard/purge-replay-sessions.ps1
+55
View File
@@ -162,6 +162,61 @@ if (Test-Path $existingManifestPath) {
}
}
# Step 3b: Recover orphaned session files that exist on disk but not in any manifest.
# This prevents a cascading failure where a corrupted or empty manifest causes
# session files to become invisible, leading to data loss on subsequent merges.
$knownUrls = [System.Collections.Generic.HashSet[string]]::new(
[string[]]@($allSessions | ForEach-Object { $_.url -replace '^sessions/', '' })
)
$orphanedFiles = Get-ChildItem -Path $sessionsWorkDir -Recurse -File -Filter '*.jsonl' -ErrorAction SilentlyContinue
$orphanedCount = 0
foreach ($file in $orphanedFiles) {
$relPath = $file.FullName.Substring($sessionsWorkDir.Length).TrimStart([IO.Path]::DirectorySeparatorChar, [IO.Path]::AltDirectorySeparatorChar)
if ($knownUrls.Contains($relPath)) { continue }
# Determine source/tags from path structure: <source>/<subdir>/<plugin>/<filename>.jsonl
$parts = $relPath -split '[/\\]'
if ($parts.Count -lt 3) { continue }
$source = $parts[0] # 'pr' or 'scheduled'
$plugin = $parts[-2] # plugin name (parent of file)
$fileName = [IO.Path]::GetFileNameWithoutExtension($parts[-1])
# Parse filename: <scenario>--<role>--run<N>
$fileParts = $fileName -split '--'
if ($fileParts.Count -lt 3) { continue }
$roleTag = $fileParts[-2]
$safeScenario = ($fileParts[0..($fileParts.Count - 3)] -join '--')
$sessionUrl = "sessions/$relPath"
$sessionId = ($relPath -replace '\.jsonl$', '')
$tags = @($source, $plugin, $roleTag, $safeScenario)
if ($source -eq 'pr' -and $parts.Count -ge 3 -and $parts[1] -match '^\d+$') {
$tags += "pr-$($parts[1])"
}
if ($source -eq 'scheduled' -and $parts.Count -ge 3 -and $parts[1] -match '^\d{4}-\d{2}-\d{2}$') {
$tags += $parts[1]
}
$displayName = "$plugin / $safeScenario ($roleTag)"
$mtime = [long]([DateTimeOffset]::new($file.LastWriteTimeUtc, [TimeSpan]::Zero).ToUnixTimeMilliseconds())
$allSessions += @{
id = $sessionId
name = $displayName
url = $sessionUrl
tags = $tags
mtime = $mtime
}
$orphanedCount++
}
if ($orphanedCount -gt 0) {
Write-Host "Recovered $orphanedCount orphaned session file(s) missing from manifest"
}
# Step 4: Write merged manifest
# Derive generated timestamp from newest session mtime to avoid gratuitous commits
# when sessions haven't changed.
eng/skill-validator/src/Evaluate/AgentRunner.cs
+4
View File
@@ -432,6 +432,10 @@ public static class AgentRunner
McpServers = sdkMcp,
CustomAgents = customAgents,
InfiniteSessions = new InfiniteSessionConfig { Enabled = false },
// SDK 0.2.x removed the built-in local-filesystem session-state
// handler. Without this, events.jsonl files are never written and
// session replay data is lost.
CreateSessionFsHandler = _ => new LocalSessionFsHandler(configDir),
OnPermissionRequest = (request, _) =>
{
// SDK 0.2.0: PermissionRequest only has Kind, no path data.
eng/skill-validator/src/Evaluate/LocalSessionFsHandler.cs
+158
View File
@@ -0,0 +1,158 @@
using GitHub.Copilot.SDK.Rpc;
namespace SkillValidator.Evaluate;
/// <summary>
/// A local-filesystem implementation of <see cref="ISessionFsHandler"/> that
/// maps SDK session-state I/O requests to physical files under a given root
/// directory. Required since Copilot SDK 0.2.x no longer ships a built-in
/// default; without this handler, <c>events.jsonl</c> files are never written.
/// </summary>
internal sealed class LocalSessionFsHandler : ISessionFsHandler
{
private readonly string _rootDir;
public LocalSessionFsHandler(string rootDir)
{
_rootDir = Path.GetFullPath(rootDir);
if (!Path.EndsInDirectorySeparator(_rootDir))
_rootDir += Path.DirectorySeparatorChar;
Directory.CreateDirectory(_rootDir);
}
/// <summary>Resolve an SDK-provided path to an absolute local path, guarding against traversal.</summary>
private string ResolvePath(string relativePath)
{
var full = Path.GetFullPath(Path.Combine(_rootDir, relativePath));
if (!full.StartsWith(_rootDir, StringComparison.OrdinalIgnoreCase))
throw new UnauthorizedAccessException($"Path traversal blocked: {relativePath}");
return full;
}
public async Task<SessionFsReadFileResult> ReadFileAsync(SessionFsReadFileParams request, CancellationToken cancellationToken)
{
var path = ResolvePath(request.Path);
var content = await File.ReadAllTextAsync(path, cancellationToken);
return new SessionFsReadFileResult { Content = content };
}
public async Task WriteFileAsync(SessionFsWriteFileParams request, CancellationToken cancellationToken)
{
var path = ResolvePath(request.Path);
var dir = Path.GetDirectoryName(path);
if (dir is not null) Directory.CreateDirectory(dir);
await File.WriteAllTextAsync(path, request.Content, cancellationToken);
}
public async Task AppendFileAsync(SessionFsAppendFileParams request, CancellationToken cancellationToken)
{
var path = ResolvePath(request.Path);
var dir = Path.GetDirectoryName(path);
if (dir is not null) Directory.CreateDirectory(dir);
await File.AppendAllTextAsync(path, request.Content, cancellationToken);
}
public Task<SessionFsExistsResult> ExistsAsync(SessionFsExistsParams request, CancellationToken cancellationToken)
{
var path = ResolvePath(request.Path);
var exists = File.Exists(path) || Directory.Exists(path);
return Task.FromResult(new SessionFsExistsResult { Exists = exists });
}
public Task<SessionFsStatResult> StatAsync(SessionFsStatParams request, CancellationToken cancellationToken)
{
var path = ResolvePath(request.Path);
if (File.Exists(path))
{
var info = new FileInfo(path);
return Task.FromResult(new SessionFsStatResult
{
IsFile = true,
IsDirectory = false,
Size = info.Length,
Mtime = info.LastWriteTimeUtc.ToString("O"),
Birthtime = info.CreationTimeUtc.ToString("O"),
});
}
if (Directory.Exists(path))
{
var info = new DirectoryInfo(path);
return Task.FromResult(new SessionFsStatResult
{
IsFile = false,
IsDirectory = true,
Size = 0,
Mtime = info.LastWriteTimeUtc.ToString("O"),
Birthtime = info.CreationTimeUtc.ToString("O"),
});
}
throw new FileNotFoundException($"Not found: {request.Path}");
}
public Task MkdirAsync(SessionFsMkdirParams request, CancellationToken cancellationToken)
{
var path = ResolvePath(request.Path);
Directory.CreateDirectory(path);
return Task.CompletedTask;
}
public Task<SessionFsReaddirResult> ReaddirAsync(SessionFsReaddirParams request, CancellationToken cancellationToken)
{
var path = ResolvePath(request.Path);
var entries = new List<string>();
if (Directory.Exists(path))
{
foreach (var entry in Directory.EnumerateFileSystemEntries(path))
entries.Add(Path.GetFileName(entry));
}
return Task.FromResult(new SessionFsReaddirResult { Entries = entries });
}
public Task<SessionFsReaddirWithTypesResult> ReaddirWithTypesAsync(SessionFsReaddirWithTypesParams request, CancellationToken cancellationToken)
{
var path = ResolvePath(request.Path);
var entries = new List<Entry>();
if (Directory.Exists(path))
{
foreach (var entry in new DirectoryInfo(path).EnumerateFileSystemInfos())
{
entries.Add(new Entry
{
Name = entry.Name,
Type = entry is DirectoryInfo ? EntryType.Directory : EntryType.File,
});
}
}
return Task.FromResult(new SessionFsReaddirWithTypesResult { Entries = entries });
}
public Task RmAsync(SessionFsRmParams request, CancellationToken cancellationToken)
{
var path = ResolvePath(request.Path);
if (File.Exists(path))
{
File.Delete(path);
}
else if (Directory.Exists(path))
{
Directory.Delete(path, recursive: request.Recursive ?? false);
}
return Task.CompletedTask;
}
public Task RenameAsync(SessionFsRenameParams request, CancellationToken cancellationToken)
{
var src = ResolvePath(request.Src);
var dest = ResolvePath(request.Dest);
var destDir = Path.GetDirectoryName(dest);
if (destDir is not null) Directory.CreateDirectory(destDir);
if (File.Exists(src))
File.Move(src, dest, overwrite: true);
else if (Directory.Exists(src))
Directory.Move(src, dest);
return Task.CompletedTask;
}
}
eng/skill-validator/src/Evaluate/SessionDatabase.cs
+19 -1
View File
@@ -296,7 +296,25 @@ public sealed class SessionDatabase : IDisposable
public void Dispose()
{
_connection.Dispose();
lock (_lock)
{
// Checkpoint WAL to ensure all data is written to the main database file.
// This is critical because the database may be packaged into artifacts
// and read by external tools (e.g., build-replay-sessions.ps1) that
// rely on the main file being up-to-date.
try
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = "PRAGMA wal_checkpoint(TRUNCATE)";
cmd.ExecuteNonQuery();
}
catch (Exception ex)
{
Console.Error.WriteLine($"Warning: WAL checkpoint failed during dispose: {ex.Message}");
}
_connection.Dispose();
}
}
}