Speed things up

This commit is contained in:
Armin Ronacher
2025-12-29 23:59:24 +01:00
parent 9094082a44
commit e1606f1ef2
3 changed files with 551 additions and 66 deletions
+60 -33
View File
@@ -120,16 +120,18 @@ func (a *App) Pull(ctx context.Context, opts PullOptions, args []string) error {
client := ghcli.NewClient(a.Runner, repoSlug(cfg))
t := a.Theme
// Fetch label colors for nice output
labelColors := a.fetchLabelColors(ctx, client)
localIssues, err := loadLocalIssues(p)
if err != nil {
return err
}
var remoteIssues []issue.Issue
var labelColors map[string]string
if len(args) > 0 {
// Fetch specific issues by number
labelColors = a.fetchLabelColors(ctx, client)
for _, arg := range args {
number := strings.TrimSpace(arg)
if number == "" {
@@ -141,52 +143,77 @@ func (a *App) Pull(ctx context.Context, opts PullOptions, args []string) error {
}
remoteIssues = append(remoteIssues, remote)
}
// Enrich with relationships
if err := client.EnrichWithRelationshipsBatch(ctx, remoteIssues); err != nil {
fmt.Fprintf(a.Err, "%s fetching relationships: %v\n", t.WarningText("Warning:"), err)
}
} else {
state := "open"
if opts.All {
state = "all"
}
remoteIssues, err = client.ListIssues(ctx, state, opts.Label)
if err != nil {
return err
// Collect issue numbers we need to fetch for closed issues
var toFetch []string
if !opts.All {
// We don't know remote issue numbers yet, so we'll collect all local non-local issues
// and filter after we get the open issues
for _, local := range localIssues {
if !local.Issue.Number.IsLocal() {
toFetch = append(toFetch, local.Issue.Number.String())
}
}
}
// When not fetching all issues, also check local open issues that might
// have been closed remotely. Build a set of already-fetched issue numbers.
if !opts.All {
// Run both queries in parallel
type listResult struct {
result ghcli.ListIssuesResult
err error
}
type batchResult struct {
issues map[string]issue.Issue
err error
}
listCh := make(chan listResult, 1)
batchCh := make(chan batchResult, 1)
go func() {
r, e := client.ListIssuesWithRelationships(ctx, state, opts.Label)
listCh <- listResult{r, e}
}()
go func() {
if len(toFetch) > 0 {
r, e := client.GetIssuesBatch(ctx, toFetch)
batchCh <- batchResult{r, e}
} else {
batchCh <- batchResult{nil, nil}
}
}()
listRes := <-listCh
if listRes.err != nil {
return listRes.err
}
remoteIssues = listRes.result.Issues
labelColors = listRes.result.LabelColors
batchRes := <-batchCh
if batchRes.err == nil && len(batchRes.issues) > 0 {
// Filter out issues we already have from the open list
fetched := make(map[string]struct{}, len(remoteIssues))
for _, ri := range remoteIssues {
fetched[ri.Number.String()] = struct{}{}
}
for _, local := range localIssues {
// Skip local-only issues (not yet pushed)
if local.Issue.Number.IsLocal() {
continue
for num, iss := range batchRes.issues {
if _, ok := fetched[num]; !ok {
remoteIssues = append(remoteIssues, iss)
}
// Skip issues we already fetched
if _, ok := fetched[local.Issue.Number.String()]; ok {
continue
}
// Fetch this issue to check if it was closed remotely
remote, err := client.GetIssue(ctx, local.Issue.Number.String())
if err != nil {
// Issue might have been deleted; skip it
continue
}
remoteIssues = append(remoteIssues, remote)
}
}
}
// Enrich all remote issues with parent/blocking relationships via GraphQL
for i := range remoteIssues {
if err := client.EnrichWithRelationships(ctx, &remoteIssues[i]); err != nil {
// Log but don't fail - relationships are optional
fmt.Fprintf(a.Err, "%s fetching relationships for #%s: %v\n",
t.WarningText("Warning:"), remoteIssues[i].Number, err)
}
}
localIssues, err = loadLocalIssues(p)
if err != nil {
return err
+409
View File
@@ -107,6 +107,221 @@ func (c *Client) ListIssues(ctx context.Context, state string, labels []string)
return issues, nil
}
// ListIssuesResult contains the result of ListIssuesWithRelationships
type ListIssuesResult struct {
Issues []issue.Issue
LabelColors map[string]string
}
// ListIssuesWithRelationships fetches issues with their relationships and label colors
// using GraphQL with pagination. This is much faster than separate calls.
func (c *Client) ListIssuesWithRelationships(ctx context.Context, state string, labels []string) (ListIssuesResult, error) {
owner, repo := splitRepo(c.repo)
if owner == "" || repo == "" {
return ListIssuesResult{}, fmt.Errorf("invalid repository format")
}
// Map state to GraphQL enum
stateFilter := "OPEN"
if state == "closed" {
stateFilter = "CLOSED"
} else if state == "all" {
stateFilter = ""
}
// Build label filter
labelFilter := ""
if len(labels) > 0 {
quoted := make([]string, len(labels))
for i, l := range labels {
quoted[i] = fmt.Sprintf("%q", l)
}
labelFilter = fmt.Sprintf(", labels: [%s]", strings.Join(quoted, ", "))
}
stateArg := ""
if stateFilter != "" {
stateArg = fmt.Sprintf(", states: [%s]", stateFilter)
}
result := ListIssuesResult{
LabelColors: make(map[string]string),
}
// Paginate through issues, fetching labels on first page
var cursor *string
firstPage := true
for {
cursorArg := "null"
if cursor != nil {
cursorArg = fmt.Sprintf("%q", *cursor)
}
// Include labels query only on first page
labelsFragment := ""
if firstPage {
labelsFragment = `labels(first: 100) {
nodes {
name
color
}
}`
}
query := fmt.Sprintf(`query($owner: String!, $repo: String!) {
repository(owner: $owner, name: $repo) {
%s
issues(first: 100%s%s, after: %s) {
pageInfo {
hasNextPage
endCursor
}
nodes {
number
title
body
state
stateReason
labels(first: 100) { nodes { name } }
assignees(first: 100) { nodes { login } }
milestone { title }
parent { number }
blockedBy(first: 100) { nodes { number } }
blocking(first: 100) { nodes { number } }
}
}
}
}`, labelsFragment, stateArg, labelFilter, cursorArg)
args := []string{"api", "graphql",
"-f", fmt.Sprintf("query=%s", query),
"-F", fmt.Sprintf("owner=%s", owner),
"-F", fmt.Sprintf("repo=%s", repo),
}
out, err := c.runner.Run(ctx, "gh", args...)
if err != nil {
return ListIssuesResult{}, err
}
var resp struct {
Data struct {
Repository struct {
Labels struct {
Nodes []struct {
Name string `json:"name"`
Color string `json:"color"`
} `json:"nodes"`
} `json:"labels"`
Issues struct {
PageInfo struct {
HasNextPage bool `json:"hasNextPage"`
EndCursor string `json:"endCursor"`
} `json:"pageInfo"`
Nodes []struct {
Number int `json:"number"`
Title string `json:"title"`
Body string `json:"body"`
State string `json:"state"`
StateReason *string `json:"stateReason"`
Labels struct {
Nodes []struct {
Name string `json:"name"`
} `json:"nodes"`
} `json:"labels"`
Assignees struct {
Nodes []struct {
Login string `json:"login"`
} `json:"nodes"`
} `json:"assignees"`
Milestone *struct {
Title string `json:"title"`
} `json:"milestone"`
Parent *struct {
Number int `json:"number"`
} `json:"parent"`
BlockedBy struct {
Nodes []struct {
Number int `json:"number"`
} `json:"nodes"`
} `json:"blockedBy"`
Blocking struct {
Nodes []struct {
Number int `json:"number"`
} `json:"nodes"`
} `json:"blocking"`
} `json:"nodes"`
} `json:"issues"`
} `json:"repository"`
} `json:"data"`
Errors []struct {
Message string `json:"message"`
} `json:"errors"`
}
if err := json.Unmarshal([]byte(out), &resp); err != nil {
return ListIssuesResult{}, fmt.Errorf("failed to parse GraphQL response: %w", err)
}
if len(resp.Errors) > 0 {
return ListIssuesResult{}, fmt.Errorf("GraphQL error: %s", resp.Errors[0].Message)
}
// Parse labels from first page
if firstPage {
for _, l := range resp.Data.Repository.Labels.Nodes {
result.LabelColors[l.Name] = l.Color
}
firstPage = false
}
for _, node := range resp.Data.Repository.Issues.Nodes {
issLabels := make([]string, 0, len(node.Labels.Nodes))
for _, l := range node.Labels.Nodes {
issLabels = append(issLabels, l.Name)
}
assignees := make([]string, 0, len(node.Assignees.Nodes))
for _, a := range node.Assignees.Nodes {
assignees = append(assignees, a.Login)
}
milestone := ""
if node.Milestone != nil {
milestone = node.Milestone.Title
}
iss := issue.Issue{
Number: issue.IssueNumber(strconv.Itoa(node.Number)),
Title: node.Title,
Body: node.Body,
State: strings.ToLower(node.State),
StateReason: node.StateReason,
Labels: issLabels,
Assignees: assignees,
Milestone: milestone,
}
if node.Parent != nil {
ref := issue.IssueRef(strconv.Itoa(node.Parent.Number))
iss.Parent = &ref
}
for _, b := range node.BlockedBy.Nodes {
iss.BlockedBy = append(iss.BlockedBy, issue.IssueRef(strconv.Itoa(b.Number)))
}
for _, b := range node.Blocking.Nodes {
iss.Blocks = append(iss.Blocks, issue.IssueRef(strconv.Itoa(b.Number)))
}
result.Issues = append(result.Issues, iss)
}
if !resp.Data.Repository.Issues.PageInfo.HasNextPage {
break
}
cursor = &resp.Data.Repository.Issues.PageInfo.EndCursor
}
return result, nil
}
// EnrichWithRelationships fetches parent and blocking relationships for an issue via GraphQL
// and updates the issue in place.
func (c *Client) EnrichWithRelationships(ctx context.Context, iss *issue.Issue) error {
@@ -126,6 +341,41 @@ func (c *Client) EnrichWithRelationships(ctx context.Context, iss *issue.Issue)
return nil
}
// EnrichWithRelationshipsBatch fetches parent and blocking relationships for multiple issues
// in a single API call and updates each issue in place.
func (c *Client) EnrichWithRelationshipsBatch(ctx context.Context, issues []issue.Issue) error {
// Collect issue numbers for non-local issues
var numbers []string
for i := range issues {
if !issues[i].Number.IsLocal() {
numbers = append(numbers, issues[i].Number.String())
}
}
if len(numbers) == 0 {
return nil
}
// Fetch all relationships in one call
rels, err := c.GetIssueRelationshipsBatch(ctx, numbers)
if err != nil {
// Don't fail if relationships can't be fetched (e.g., feature not available)
return nil
}
// Apply relationships to each issue
for i := range issues {
num := issues[i].Number.String()
if rel, ok := rels[num]; ok {
issues[i].Parent = rel.Parent
issues[i].BlockedBy = rel.BlockedBy
issues[i].Blocks = rel.Blocks
}
}
return nil
}
func (c *Client) GetIssue(ctx context.Context, number string) (issue.Issue, error) {
args := []string{"issue", "view", number, "--json", "number,title,body,labels,assignees,milestone,state,stateReason"}
out, err := c.runner.Run(ctx, "gh", c.withRepo(args)...)
@@ -139,6 +389,165 @@ func (c *Client) GetIssue(ctx context.Context, number string) (issue.Issue, erro
return payload.ToIssue(), nil
}
// GetIssuesBatch fetches multiple issues in a single GraphQL call.
// Returns a map of issue number -> issue. Issues that don't exist are not included.
func (c *Client) GetIssuesBatch(ctx context.Context, numbers []string) (map[string]issue.Issue, error) {
if len(numbers) == 0 {
return map[string]issue.Issue{}, nil
}
owner, repo := splitRepo(c.repo)
if owner == "" || repo == "" {
return nil, fmt.Errorf("invalid repository format")
}
// Build a batched GraphQL query with aliases for each issue
var issueQueries []string
for i, num := range numbers {
n, err := strconv.Atoi(num)
if err != nil {
continue
}
issueQueries = append(issueQueries, fmt.Sprintf(`issue%d: issue(number: %d) {
number
title
body
state
stateReason
labels(first: 100) { nodes { name } }
assignees(first: 100) { nodes { login } }
milestone { title }
parent { number }
blockedBy(first: 100) { nodes { number } }
blocking(first: 100) { nodes { number } }
}`, i, n))
}
if len(issueQueries) == 0 {
return map[string]issue.Issue{}, nil
}
query := fmt.Sprintf(`query($owner: String!, $repo: String!) {
repository(owner: $owner, name: $repo) {
%s
}
}`, strings.Join(issueQueries, "\n "))
args := []string{"api", "graphql",
"-f", fmt.Sprintf("query=%s", query),
"-F", fmt.Sprintf("owner=%s", owner),
"-F", fmt.Sprintf("repo=%s", repo),
}
out, err := c.runner.Run(ctx, "gh", args...)
if err != nil {
return nil, err
}
var resp struct {
Data struct {
Repository map[string]json.RawMessage `json:"repository"`
} `json:"data"`
Errors []struct {
Message string `json:"message"`
} `json:"errors"`
}
if err := json.Unmarshal([]byte(out), &resp); err != nil {
return nil, fmt.Errorf("failed to parse GraphQL response: %w", err)
}
if len(resp.Errors) > 0 {
return nil, fmt.Errorf("GraphQL error: %s", resp.Errors[0].Message)
}
results := make(map[string]issue.Issue)
for alias, rawIssue := range resp.Data.Repository {
if !strings.HasPrefix(alias, "issue") {
continue
}
if string(rawIssue) == "null" {
continue
}
var issueData struct {
Number int `json:"number"`
Title string `json:"title"`
Body string `json:"body"`
State string `json:"state"`
StateReason *string `json:"stateReason"`
Labels struct {
Nodes []struct {
Name string `json:"name"`
} `json:"nodes"`
} `json:"labels"`
Assignees struct {
Nodes []struct {
Login string `json:"login"`
} `json:"nodes"`
} `json:"assignees"`
Milestone *struct {
Title string `json:"title"`
} `json:"milestone"`
Parent *struct {
Number int `json:"number"`
} `json:"parent"`
BlockedBy struct {
Nodes []struct {
Number int `json:"number"`
} `json:"nodes"`
} `json:"blockedBy"`
Blocking struct {
Nodes []struct {
Number int `json:"number"`
} `json:"nodes"`
} `json:"blocking"`
}
if err := json.Unmarshal(rawIssue, &issueData); err != nil {
continue
}
labels := make([]string, 0, len(issueData.Labels.Nodes))
for _, l := range issueData.Labels.Nodes {
labels = append(labels, l.Name)
}
assignees := make([]string, 0, len(issueData.Assignees.Nodes))
for _, a := range issueData.Assignees.Nodes {
assignees = append(assignees, a.Login)
}
milestone := ""
if issueData.Milestone != nil {
milestone = issueData.Milestone.Title
}
iss := issue.Issue{
Number: issue.IssueNumber(strconv.Itoa(issueData.Number)),
Title: issueData.Title,
Body: issueData.Body,
State: strings.ToLower(issueData.State),
StateReason: issueData.StateReason,
Labels: labels,
Assignees: assignees,
Milestone: milestone,
}
if issueData.Parent != nil {
ref := issue.IssueRef(strconv.Itoa(issueData.Parent.Number))
iss.Parent = &ref
}
for _, b := range issueData.BlockedBy.Nodes {
iss.BlockedBy = append(iss.BlockedBy, issue.IssueRef(strconv.Itoa(b.Number)))
}
for _, b := range issueData.Blocking.Nodes {
iss.Blocks = append(iss.Blocks, issue.IssueRef(strconv.Itoa(b.Number)))
}
results[strconv.Itoa(issueData.Number)] = iss
}
return results, nil
}
func (c *Client) CreateIssue(ctx context.Context, issue issue.Issue) (string, error) {
args := []string{"issue", "create", "--title", issue.Title, "--body", issue.Body}
for _, label := range issue.Labels {
+82 -33
View File
@@ -59,15 +59,38 @@ type graphqlMutationResponse struct {
// GetIssueRelationships fetches parent and blocking relationships for an issue via GraphQL.
func (c *Client) GetIssueRelationships(ctx context.Context, number string) (IssueRelationships, string, error) {
owner, repo := splitRepo(c.repo)
if owner == "" || repo == "" {
return IssueRelationships{}, "", fmt.Errorf("invalid repository format")
results, err := c.GetIssueRelationshipsBatch(ctx, []string{number})
if err != nil {
return IssueRelationships{}, "", err
}
if rel, ok := results[number]; ok {
return rel, "", nil // Note: we don't return the ID anymore, but it's not used
}
return IssueRelationships{}, "", fmt.Errorf("issue %s not found in response", number)
}
// GetIssueRelationshipsBatch fetches parent and blocking relationships for multiple issues
// in a single GraphQL call. Returns a map of issue number -> relationships.
func (c *Client) GetIssueRelationshipsBatch(ctx context.Context, numbers []string) (map[string]IssueRelationships, error) {
if len(numbers) == 0 {
return map[string]IssueRelationships{}, nil
}
query := `
query($owner: String!, $repo: String!, $number: Int!) {
repository(owner: $owner, name: $repo) {
issue(number: $number) {
owner, repo := splitRepo(c.repo)
if owner == "" || repo == "" {
return nil, fmt.Errorf("invalid repository format")
}
// Build a batched GraphQL query with aliases for each issue
// GraphQL aliases allow us to fetch multiple issues in one query:
// query { repository(owner: "x", name: "y") { issue1: issue(number: 1) { ... } issue2: issue(number: 2) { ... } } }
var issueQueries []string
for i, num := range numbers {
n, err := strconv.Atoi(num)
if err != nil {
continue // Skip invalid numbers
}
issueQueries = append(issueQueries, fmt.Sprintf(`issue%d: issue(number: %d) {
id
number
parent {
@@ -86,53 +109,79 @@ query($owner: String!, $repo: String!, $number: Int!) {
id
}
}
}
}
}`
num, err := strconv.Atoi(number)
if err != nil {
return IssueRelationships{}, "", fmt.Errorf("invalid issue number: %s", number)
}`, i, n))
}
if len(issueQueries) == 0 {
return map[string]IssueRelationships{}, nil
}
query := fmt.Sprintf(`query($owner: String!, $repo: String!) {
repository(owner: $owner, name: $repo) {
%s
}
}`, strings.Join(issueQueries, "\n "))
args := []string{"api", "graphql",
"-f", fmt.Sprintf("query=%s", query),
"-F", fmt.Sprintf("owner=%s", owner),
"-F", fmt.Sprintf("repo=%s", repo),
"-F", fmt.Sprintf("number=%d", num),
}
out, err := c.runner.Run(ctx, "gh", args...)
if err != nil {
return IssueRelationships{}, "", err
return nil, err
}
var resp graphqlResponse
// Parse the response - we need a dynamic structure since aliases are dynamic
var resp struct {
Data struct {
Repository map[string]json.RawMessage `json:"repository"`
} `json:"data"`
Errors []struct {
Message string `json:"message"`
} `json:"errors"`
}
if err := json.Unmarshal([]byte(out), &resp); err != nil {
return IssueRelationships{}, "", fmt.Errorf("failed to parse GraphQL response: %w", err)
return nil, fmt.Errorf("failed to parse GraphQL response: %w", err)
}
if len(resp.Errors) > 0 {
return IssueRelationships{}, "", fmt.Errorf("GraphQL error: %s", resp.Errors[0].Message)
return nil, fmt.Errorf("GraphQL error: %s", resp.Errors[0].Message)
}
issueData := resp.Data.Repository.Issue
rels := IssueRelationships{}
results := make(map[string]IssueRelationships)
if issueData.Parent != nil {
ref := issue.IssueRef(strconv.Itoa(issueData.Parent.Number))
rels.Parent = &ref
// Parse each aliased issue response
for alias, rawIssue := range resp.Data.Repository {
if !strings.HasPrefix(alias, "issue") {
continue
}
if string(rawIssue) == "null" {
continue
}
var issueData graphqlIssue
if err := json.Unmarshal(rawIssue, &issueData); err != nil {
continue // Skip malformed issues
}
rels := IssueRelationships{}
if issueData.Parent != nil {
ref := issue.IssueRef(strconv.Itoa(issueData.Parent.Number))
rels.Parent = &ref
}
for _, node := range issueData.BlockedBy.Nodes {
rels.BlockedBy = append(rels.BlockedBy, issue.IssueRef(strconv.Itoa(node.Number)))
}
for _, node := range issueData.Blocking.Nodes {
rels.Blocks = append(rels.Blocks, issue.IssueRef(strconv.Itoa(node.Number)))
}
results[strconv.Itoa(issueData.Number)] = rels
}
for _, node := range issueData.BlockedBy.Nodes {
rels.BlockedBy = append(rels.BlockedBy, issue.IssueRef(strconv.Itoa(node.Number)))
}
for _, node := range issueData.Blocking.Nodes {
rels.Blocks = append(rels.Blocks, issue.IssueRef(strconv.Itoa(node.Number)))
}
return rels, issueData.ID, nil
return results, nil
}
// GetIssueNodeID fetches the GraphQL node ID for an issue.