From 05e2868b007cb3014513f1777cf81c44faf4f4f4 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 17 Apr 2026 14:58:15 +0200 Subject: [PATCH] Reintroduce copilot sessions storing (#530) --- .github/workflows/evaluation.yml | 2 +- eng/dashboard/build-replay-sessions.ps1 | 39 ++++- eng/dashboard/purge-replay-sessions.ps1 | 55 ++++++ .../src/Evaluate/AgentRunner.cs | 4 + .../src/Evaluate/LocalSessionFsHandler.cs | 158 ++++++++++++++++++ .../src/Evaluate/SessionDatabase.cs | 20 ++- 6 files changed, 271 insertions(+), 7 deletions(-) create mode 100644 eng/skill-validator/src/Evaluate/LocalSessionFsHandler.cs diff --git a/.github/workflows/evaluation.yml b/.github/workflows/evaluation.yml index 4ff1a56f..e0f5ab47 100644 --- a/.github/workflows/evaluation.yml +++ b/.github/workflows/evaluation.yml @@ -1197,7 +1197,7 @@ jobs: !cancelled() && needs.discover.result == 'success' && needs.discover.outputs.has_plugins == 'true' && - github.ref == 'refs/heads/main' + (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main')) concurrency: group: deploy-dashboard cancel-in-progress: false diff --git a/eng/dashboard/build-replay-sessions.ps1 b/eng/dashboard/build-replay-sessions.ps1 index e2250c6a..7e5ab4bd 100644 --- a/eng/dashboard/build-replay-sessions.ps1 +++ b/eng/dashboard/build-replay-sessions.ps1 @@ -143,6 +143,13 @@ $manifestSessions = @() $artifactDirs = Get-ChildItem -Path $ResultsDir -Directory -ErrorAction SilentlyContinue | Where-Object { $_.Name -like 'skill-validator-results-*' } +Write-Host "Scanning $ResultsDir for session artifacts..." +$allDirs = @(Get-ChildItem -Path $ResultsDir -Directory -ErrorAction SilentlyContinue) +Write-Host " Total directories in ResultsDir: $($allDirs.Count)" +if ($allDirs.Count -gt 0) { + Write-Host " Directory names: $($allDirs.Name -join ', ')" +} + if (-not $artifactDirs) { Write-Warning "No skill-validator-results-* directories found in $ResultsDir" # Write empty manifest @@ -159,23 +166,30 @@ foreach ($artifactDir in $artifactDirs) { $pluginName = ($entryName -split '--')[0] # Find timestamped result directory - $runDir = Get-ChildItem -Path $artifactDir.FullName -Directory -ErrorAction SilentlyContinue | + $allSubDirs = @(Get-ChildItem -Path $artifactDir.FullName -Directory -ErrorAction SilentlyContinue) + $runDir = $allSubDirs | Where-Object { $_.Name -match '^\d{8}-\d{6}$' } | Sort-Object Name -Descending | Select-Object -First 1 if (-not $runDir) { - Write-Warning "No timestamped run directory found in $($artifactDir.Name), skipping" + Write-Warning "No timestamped run directory found in $($artifactDir.Name), skipping (subdirs: $($allSubDirs.Name -join ', '))" continue } $sessionsDbPath = Join-Path $runDir.FullName "sessions.db" if (-not (Test-Path $sessionsDbPath)) { - Write-Warning "No sessions.db found in $($runDir.FullName), skipping" + # List files in the run directory for diagnostics + $runFiles = Get-ChildItem -Path $runDir.FullName -File -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Name + Write-Warning "No sessions.db found in $($runDir.FullName), skipping (files: $($runFiles -join ', '))" continue } - Write-Host "Processing sessions from $($artifactDir.Name) ($($runDir.Name))..." + # Check for WAL file alongside the database + $walPath = "$sessionsDbPath-wal" + $hasWal = Test-Path $walPath + $dbSize = (Get-Item $sessionsDbPath).Length + Write-Host "Processing sessions from $($artifactDir.Name) ($($runDir.Name))... [db=$([math]::Round($dbSize/1024,1))KB, wal=$hasWal]" # Query sessions.db $query = "SELECT id, skill_name, scenario_name, role, run_index, model, status, config_dir FROM sessions WHERE status IN ('completed', 'timed_out') ORDER BY skill_name, scenario_name, run_index, role;" @@ -187,6 +201,17 @@ foreach ($artifactDir in $artifactDirs) { continue } + Write-Host " Found $($rows.Count) completed/timed_out session(s) in DB" + if ($rows.Count -eq 0) { + # Query total sessions (any status) for diagnostics + try { + $allRows = Invoke-SqliteQuery -DatabasePath $sessionsDbPath -Query "SELECT id, status FROM sessions;" + Write-Warning " DB has $($allRows.Count) total session(s) with statuses: $(($allRows | ForEach-Object { ($_ -split '\|')[1] } | Sort-Object -Unique) -join ', ')" + } catch { + Write-Warning " Could not query all sessions: $_" + } + } + foreach ($row in $rows) { if (-not $row) { continue } $fields = $row -split '\|' @@ -234,7 +259,7 @@ foreach ($artifactDir in $artifactDirs) { $displayName = "$pluginName / $scenarioName ($roleTag, run $runIndex)" $id = "$subDir/$pluginName/$safeScenario--$roleTag--run$runIndex" - $tags = @($Source, $pluginName, $roleTag) + $tags = @($Source, $pluginName, $roleTag, $safeScenario) if ($Source -eq 'pr' -and $PrNumber -gt 0) { $tags += "pr-$PrNumber" } @@ -264,3 +289,7 @@ $manifestPath = Join-Path $OutputDir "manifest.json" $manifest | ConvertTo-Json -Depth 10 | Out-File -FilePath $manifestPath -Encoding utf8 Write-Host "`nManifest written to $manifestPath with $($manifestSessions.Count) session(s)" +if ($manifestSessions.Count -eq 0) { + Write-Warning "No sessions were extracted from any artifact. Check warnings above for details." + Write-Warning "Artifact dirs scanned: $($artifactDirs.Count)" +} diff --git a/eng/dashboard/purge-replay-sessions.ps1 b/eng/dashboard/purge-replay-sessions.ps1 index 990236df..4636e340 100644 --- a/eng/dashboard/purge-replay-sessions.ps1 +++ b/eng/dashboard/purge-replay-sessions.ps1 @@ -162,6 +162,61 @@ if (Test-Path $existingManifestPath) { } } +# Step 3b: Recover orphaned session files that exist on disk but not in any manifest. +# This prevents a cascading failure where a corrupted or empty manifest causes +# session files to become invisible, leading to data loss on subsequent merges. +$knownUrls = [System.Collections.Generic.HashSet[string]]::new( + [string[]]@($allSessions | ForEach-Object { $_.url -replace '^sessions/', '' }) +) +$orphanedFiles = Get-ChildItem -Path $sessionsWorkDir -Recurse -File -Filter '*.jsonl' -ErrorAction SilentlyContinue +$orphanedCount = 0 +foreach ($file in $orphanedFiles) { + $relPath = $file.FullName.Substring($sessionsWorkDir.Length).TrimStart([IO.Path]::DirectorySeparatorChar, [IO.Path]::AltDirectorySeparatorChar) + if ($knownUrls.Contains($relPath)) { continue } + + # Determine source/tags from path structure: ///.jsonl + $parts = $relPath -split '[/\\]' + if ($parts.Count -lt 3) { continue } + + $source = $parts[0] # 'pr' or 'scheduled' + $plugin = $parts[-2] # plugin name (parent of file) + $fileName = [IO.Path]::GetFileNameWithoutExtension($parts[-1]) + + # Parse filename: ----run + $fileParts = $fileName -split '--' + if ($fileParts.Count -lt 3) { continue } + + $roleTag = $fileParts[-2] + $safeScenario = ($fileParts[0..($fileParts.Count - 3)] -join '--') + + $sessionUrl = "sessions/$relPath" + $sessionId = ($relPath -replace '\.jsonl$', '') + + $tags = @($source, $plugin, $roleTag, $safeScenario) + if ($source -eq 'pr' -and $parts.Count -ge 3 -and $parts[1] -match '^\d+$') { + $tags += "pr-$($parts[1])" + } + if ($source -eq 'scheduled' -and $parts.Count -ge 3 -and $parts[1] -match '^\d{4}-\d{2}-\d{2}$') { + $tags += $parts[1] + } + + $displayName = "$plugin / $safeScenario ($roleTag)" + $mtime = [long]([DateTimeOffset]::new($file.LastWriteTimeUtc, [TimeSpan]::Zero).ToUnixTimeMilliseconds()) + + $allSessions += @{ + id = $sessionId + name = $displayName + url = $sessionUrl + tags = $tags + mtime = $mtime + } + $orphanedCount++ +} + +if ($orphanedCount -gt 0) { + Write-Host "Recovered $orphanedCount orphaned session file(s) missing from manifest" +} + # Step 4: Write merged manifest # Derive generated timestamp from newest session mtime to avoid gratuitous commits # when sessions haven't changed. diff --git a/eng/skill-validator/src/Evaluate/AgentRunner.cs b/eng/skill-validator/src/Evaluate/AgentRunner.cs index 04c78d44..911f1298 100644 --- a/eng/skill-validator/src/Evaluate/AgentRunner.cs +++ b/eng/skill-validator/src/Evaluate/AgentRunner.cs @@ -432,6 +432,10 @@ public static class AgentRunner McpServers = sdkMcp, CustomAgents = customAgents, InfiniteSessions = new InfiniteSessionConfig { Enabled = false }, + // SDK 0.2.x removed the built-in local-filesystem session-state + // handler. Without this, events.jsonl files are never written and + // session replay data is lost. + CreateSessionFsHandler = _ => new LocalSessionFsHandler(configDir), OnPermissionRequest = (request, _) => { // SDK 0.2.0: PermissionRequest only has Kind, no path data. diff --git a/eng/skill-validator/src/Evaluate/LocalSessionFsHandler.cs b/eng/skill-validator/src/Evaluate/LocalSessionFsHandler.cs new file mode 100644 index 00000000..ce724f81 --- /dev/null +++ b/eng/skill-validator/src/Evaluate/LocalSessionFsHandler.cs @@ -0,0 +1,158 @@ +using GitHub.Copilot.SDK.Rpc; + +namespace SkillValidator.Evaluate; + +/// +/// A local-filesystem implementation of that +/// maps SDK session-state I/O requests to physical files under a given root +/// directory. Required since Copilot SDK 0.2.x no longer ships a built-in +/// default; without this handler, events.jsonl files are never written. +/// +internal sealed class LocalSessionFsHandler : ISessionFsHandler +{ + private readonly string _rootDir; + + public LocalSessionFsHandler(string rootDir) + { + _rootDir = Path.GetFullPath(rootDir); + if (!Path.EndsInDirectorySeparator(_rootDir)) + _rootDir += Path.DirectorySeparatorChar; + Directory.CreateDirectory(_rootDir); + } + + /// Resolve an SDK-provided path to an absolute local path, guarding against traversal. + private string ResolvePath(string relativePath) + { + var full = Path.GetFullPath(Path.Combine(_rootDir, relativePath)); + if (!full.StartsWith(_rootDir, StringComparison.OrdinalIgnoreCase)) + throw new UnauthorizedAccessException($"Path traversal blocked: {relativePath}"); + return full; + } + + public async Task ReadFileAsync(SessionFsReadFileParams request, CancellationToken cancellationToken) + { + var path = ResolvePath(request.Path); + var content = await File.ReadAllTextAsync(path, cancellationToken); + return new SessionFsReadFileResult { Content = content }; + } + + public async Task WriteFileAsync(SessionFsWriteFileParams request, CancellationToken cancellationToken) + { + var path = ResolvePath(request.Path); + var dir = Path.GetDirectoryName(path); + if (dir is not null) Directory.CreateDirectory(dir); + await File.WriteAllTextAsync(path, request.Content, cancellationToken); + } + + public async Task AppendFileAsync(SessionFsAppendFileParams request, CancellationToken cancellationToken) + { + var path = ResolvePath(request.Path); + var dir = Path.GetDirectoryName(path); + if (dir is not null) Directory.CreateDirectory(dir); + await File.AppendAllTextAsync(path, request.Content, cancellationToken); + } + + public Task ExistsAsync(SessionFsExistsParams request, CancellationToken cancellationToken) + { + var path = ResolvePath(request.Path); + var exists = File.Exists(path) || Directory.Exists(path); + return Task.FromResult(new SessionFsExistsResult { Exists = exists }); + } + + public Task StatAsync(SessionFsStatParams request, CancellationToken cancellationToken) + { + var path = ResolvePath(request.Path); + if (File.Exists(path)) + { + var info = new FileInfo(path); + return Task.FromResult(new SessionFsStatResult + { + IsFile = true, + IsDirectory = false, + Size = info.Length, + Mtime = info.LastWriteTimeUtc.ToString("O"), + Birthtime = info.CreationTimeUtc.ToString("O"), + }); + } + + if (Directory.Exists(path)) + { + var info = new DirectoryInfo(path); + return Task.FromResult(new SessionFsStatResult + { + IsFile = false, + IsDirectory = true, + Size = 0, + Mtime = info.LastWriteTimeUtc.ToString("O"), + Birthtime = info.CreationTimeUtc.ToString("O"), + }); + } + + throw new FileNotFoundException($"Not found: {request.Path}"); + } + + public Task MkdirAsync(SessionFsMkdirParams request, CancellationToken cancellationToken) + { + var path = ResolvePath(request.Path); + Directory.CreateDirectory(path); + return Task.CompletedTask; + } + + public Task ReaddirAsync(SessionFsReaddirParams request, CancellationToken cancellationToken) + { + var path = ResolvePath(request.Path); + var entries = new List(); + if (Directory.Exists(path)) + { + foreach (var entry in Directory.EnumerateFileSystemEntries(path)) + entries.Add(Path.GetFileName(entry)); + } + return Task.FromResult(new SessionFsReaddirResult { Entries = entries }); + } + + public Task ReaddirWithTypesAsync(SessionFsReaddirWithTypesParams request, CancellationToken cancellationToken) + { + var path = ResolvePath(request.Path); + var entries = new List(); + if (Directory.Exists(path)) + { + foreach (var entry in new DirectoryInfo(path).EnumerateFileSystemInfos()) + { + entries.Add(new Entry + { + Name = entry.Name, + Type = entry is DirectoryInfo ? EntryType.Directory : EntryType.File, + }); + } + } + return Task.FromResult(new SessionFsReaddirWithTypesResult { Entries = entries }); + } + + public Task RmAsync(SessionFsRmParams request, CancellationToken cancellationToken) + { + var path = ResolvePath(request.Path); + if (File.Exists(path)) + { + File.Delete(path); + } + else if (Directory.Exists(path)) + { + Directory.Delete(path, recursive: request.Recursive ?? false); + } + return Task.CompletedTask; + } + + public Task RenameAsync(SessionFsRenameParams request, CancellationToken cancellationToken) + { + var src = ResolvePath(request.Src); + var dest = ResolvePath(request.Dest); + var destDir = Path.GetDirectoryName(dest); + if (destDir is not null) Directory.CreateDirectory(destDir); + + if (File.Exists(src)) + File.Move(src, dest, overwrite: true); + else if (Directory.Exists(src)) + Directory.Move(src, dest); + return Task.CompletedTask; + } +} diff --git a/eng/skill-validator/src/Evaluate/SessionDatabase.cs b/eng/skill-validator/src/Evaluate/SessionDatabase.cs index 6ad3f8a2..a6e4aa25 100644 --- a/eng/skill-validator/src/Evaluate/SessionDatabase.cs +++ b/eng/skill-validator/src/Evaluate/SessionDatabase.cs @@ -296,7 +296,25 @@ public sealed class SessionDatabase : IDisposable public void Dispose() { - _connection.Dispose(); + lock (_lock) + { + // Checkpoint WAL to ensure all data is written to the main database file. + // This is critical because the database may be packaged into artifacts + // and read by external tools (e.g., build-replay-sessions.ps1) that + // rely on the main file being up-to-date. + try + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "PRAGMA wal_checkpoint(TRUNCATE)"; + cmd.ExecuteNonQuery(); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Warning: WAL checkpoint failed during dispose: {ex.Message}"); + } + + _connection.Dispose(); + } } }