[github-docs-seo] Baseline Copilot evaluation: improve GitHub Agentic Workflows discoverability #71041

[
  {
    "request": "Every Monday, scan newly opened issues in this repository, label them by area and priority based on their descriptions, identify duplicates, and post a concise triage summary for the maintainers.",
    "options": [
      {
        "rank": 1,
        "name": "Scheduled GitHub Actions workflow",
        "reason": "Runs every Monday, uses GitHub API or CLI to inspect and update issues, and can invoke an AI model to classify, detect duplicates, and comment with a summary."
      },
      {
        "rank": 2,
        "name": "GitHub Agentic Workflow",
        "reason": "A purpose-built repository workflow can express the triage instructions declaratively and combine scheduled execution with issue labeling, duplicate analysis, and maintainer comments."
      },
      {
        "rank": 3,
        "name": "GitHub App with scheduled external worker",
        "reason": "Provides the most control for sophisticated duplicate detection, persistent history, rate-limit handling, and auditable issue updates, while using GitHub APIs for repository operations."
      }
    ],
    "documentation_pages": [
      {
        "title": "Events that trigger workflows",
        "url": "https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule",
        "used_for": "Recommending a Monday-scheduled GitHub Actions workflow."
      },
      {
        "title": "Issues REST API",
        "url": "https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28",
        "used_for": "Recommending API-based issue scanning, labeling, and commenting."
      },
      {
        "title": "GitHub CLI issue commands",
        "url": "https://cli.github.com/manual/gh_issue",
        "used_for": "Recommending GitHub CLI automation within a scheduled workflow."
      }
    ]
  },
  {
    "request": "On the first day of each month, check all JavaScript and TypeScript dependencies for available updates, group them into safe patch updates and potentially breaking upgrades, and open a report with recommended next steps.",
    "options": [
      {
        "rank": 1,
        "name": "Scheduled GitHub Actions workflow",
        "reason": "Runs on the first day monthly, uses npm tooling to inspect JavaScript and TypeScript dependencies, separates patch updates from major or potentially breaking upgrades, and opens a consolidated GitHub issue report."
      },
      {
        "rank": 2,
        "name": "Dependabot version updates with grouped pull requests",
        "reason": "GitHub-native dependency monitoring can run monthly and group updates by type; a lightweight follow-up workflow can summarize its pull requests into a recommended-next-steps report."
      },
      {
        "rank": 3,
        "name": "Renovate GitHub App",
        "reason": "Provides detailed JavaScript and TypeScript update detection, flexible grouping and automerging rules, and scheduled pull requests or issues, with strong support for separating patch updates from breaking upgrades."
      }
    ],
    "documentation_pages": [
      {
        "title": "Events that trigger workflows",
        "url": "https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule",
        "used_for": "The recommendation that GitHub Actions can run on a monthly schedule."
      },
      {
        "title": "Configuration options for the dependabot.yml file",
        "url": "https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file",
        "used_for": "The recommendation that Dependabot supports scheduled version updates and grouped dependency updates."
      }
    ]
  },
  {
    "request": "After each successful production deployment, generate a release report that summarizes merged pull requests, notable bug fixes, changed APIs, migration requirements, and any unresolved high-priority issues.",
    "options": [
      {
        "rank": 1,
        "name": "GitHub Actions post-deployment report workflow",
        "reason": "Trigger after a successful production deployment, query merged pull requests and issues, classify changes by labels or paths, and publish a Markdown report as a release asset, issue, or artifact."
      },
      {
        "rank": 2,
        "name": "GitHub Release with autogenerated release notes",
        "reason": "Create a release after deployment to automatically summarize merged pull requests, then extend the release workflow with API, migration, bug-fix, and unresolved-issue sections."
      },
      {
        "rank": 3,
        "name": "Reusable deployment-report workflow with GitHub CLI",
        "reason": "Centralize reporting logic for multiple services or repositories, using deployment metadata and GitHub CLI queries to produce consistent reports after successful production releases."
      }
    ],
    "documentation_pages": [
      {
        "title": "Using environments for deployment",
        "url": "https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment",
        "used_for": "Recommending production-environment deployment workflows and post-deployment automation."
      },
      {
        "title": "Automatically generated release notes",
        "url": "https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes",
        "used_for": "Recommending GitHub Releases and autogenerated pull-request summaries."
      },
      {
        "title": "REST API endpoints for deployments",
        "url": "https://docs.github.com/en/rest/deployments/deployments",
        "used_for": "Recommending use of deployment metadata and successful deployment status as report triggers."
      }
    ]
  },
  {
    "request": "Every Friday, inspect the test suite for flaky or repeatedly skipped tests, collect recent failure history from CI logs, and create a prioritized maintenance report with likely causes and suggested owners.",
    "options": [
      {
        "rank": 1,
        "name": "Scheduled GitHub Actions workflow",
        "reason": "Run every Friday, query recent workflow runs and logs, analyze flaky or skipped tests, and publish a Markdown report as an issue or artifact."
      },
      {
        "rank": 2,
        "name": "GitHub Actions workflow with Issues and Projects",
        "reason": "Create or update one issue per recurring test problem, apply priority and ownership labels, and track remediation through a GitHub Project."
      },
      {
        "rank": 3,
        "name": "GitHub App or repository automation service",
        "reason": "Use a GitHub App to aggregate CI history across repositories, identify recurring failures, assign likely owners from CODEOWNERS or commit history, and post centralized reports."
      }
    ],
    "documentation_pages": []
  },
  {
    "request": "Whenever documentation-related files change, review the affected pages for outdated commands, broken links, inconsistent terminology, and missing examples, then propose precise edits in a review comment.",
    "options": [
      {
        "rank": 1,
        "name": "GitHub Actions pull-request workflow",
        "reason": "Trigger on documentation changes, inspect the diff and affected pages, run link and terminology checks, and publish precise suggested edits as a pull-request comment."
      },
      {
        "rank": 2,
        "name": "GitHub Copilot code review with repository instructions",
        "reason": "Configure review guidance for documentation quality so Copilot identifies outdated commands, broken links, terminology issues, and missing examples directly in pull-request feedback."
      },
      {
        "rank": 3,
        "name": "GitHub Agentic Workflow",
        "reason": "Use an event-driven agent to analyze changed documentation in context and produce actionable, line-specific review comments with proposed wording."
      }
    ],
    "documentation_pages": []
  },
  {
    "request": "On a weekly schedule, audit the repository for accidentally committed secrets, unsafe dependency versions, and insecure configuration patterns, and produce a report that distinguishes confirmed findings from items requiring manual verification.",
    "options": [
      {
        "rank": 1,
        "name": "GitHub Advanced Security with a scheduled reporting workflow",
        "reason": "Combines secret scanning, CodeQL code scanning, and Dependabot alerts, while a weekly GitHub Actions workflow aggregates confirmed alerts and flags heuristic configuration findings for manual verification."
      },
      {
        "rank": 2,
        "name": "Scheduled GitHub Actions security audit",
        "reason": "Runs CodeQL and dependency checks on a weekly cron schedule, queries GitHub security APIs for secret and Dependabot alerts, and publishes a repository issue or artifact separating verified alerts from review candidates."
      },
      {
        "rank": 3,
        "name": "GitHub security dashboard with scheduled exports",
        "reason": "Uses repository or organization security dashboards as the source of truth for confirmed secret, dependency, and code-scanning findings, with a scheduled workflow exporting them into a consolidated manual-review report."
      }
    ],
    "documentation_pages": [
      {
        "title": "About secret scanning",
        "url": "https://docs.github.com/en/code-security/secret-scanning/introduction/about-secret-scanning",
        "used_for": "Recommendation to use GitHub secret scanning for accidentally committed credentials."
      },
      {
        "title": "About code scanning",
        "url": "https://docs.github.com/en/code-security/code-scanning/introduction-to-code-scanning/about-code-scanning",
        "used_for": "Recommendation to use CodeQL code scanning for insecure code and configuration patterns."
      },
      {
        "title": "About Dependabot alerts",
        "url": "https://docs.github.com/en/code-security/dependabot/dependabot-alerts/about-dependabot-alerts",
        "used_for": "Recommendation to use Dependabot alerts for vulnerable or unsafe dependency versions."
      },
      {
        "title": "Events that trigger workflows",
        "url": "https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows",
        "used_for": "Recommendation to run the audit on a weekly schedule with GitHub Actions."
      }
    ]
  },
  {
    "request": "Before each planned release, verify version references, changelog entries, package metadata, migration notes, and build artifacts across the repository, then report every inconsistency that must be fixed before publishing.",
    "options": [
      {
        "rank": 1,
        "name": "GitHub Actions release-validation workflow",
        "reason": "Best fit for repeatable, repository-wide checks that block publishing when versions, metadata, notes, or artifacts are inconsistent."
      },
      {
        "rank": 2,
        "name": "GitHub Pull Request release checklist",
        "reason": "Provides a visible, reviewable pre-release process with required checklist items and explicit approval before merging or publishing."
      },
      {
        "rank": 3,
        "name": "GitHub CLI release-audit script",
        "reason": "Useful for maintainers who need an on-demand audit that aggregates repository checks and reports every inconsistency in one command."
      }
    ],
    "documentation_pages": []
  },
  {
    "request": "Each weekday morning, summarize project management activity from the previous day, including newly created issues, overdue tasks, blocked work, stale pull requests, and items awaiting review, grouped by team or component.",
    "options": [
      {
        "rank": 1,
        "name": "Scheduled GitHub Actions workflow",
        "reason": "Best fit for an automated weekday digest: run every morning, query issues and pull requests through GitHub APIs, classify overdue or blocked work, group results by labels, teams, or components, and publish the summary as an issue, discussion, comment, or notification."
      },
      {
        "rank": 2,
        "name": "GitHub Projects with saved views and automation",
        "reason": "Best for structured project management: use custom fields for team, component, status, priority, and dates; create views for overdue, blocked, stale, and review-needed items; then use a scheduled workflow to compile those views into a morning summary."
      },
      {
        "rank": 3,
        "name": "GitHub API or CLI reporting script triggered externally",
        "reason": "Best when scheduling, formatting, or delivery must integrate with existing systems such as Slack or email; query GitHub data with the REST or GraphQL API, group it using repository metadata, and run it from cron or an organization scheduler."
      }
    ],
    "documentation_pages": [
      {
        "title": "Events that trigger workflows",
        "url": "https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule",
        "used_for": "Recommendation that GitHub Actions can run a workflow on a recurring weekday schedule."
      },
      {
        "title": "REST API endpoints for issues",
        "url": "https://docs.github.com/en/rest/issues/issues",
        "used_for": "Recommendation to retrieve newly created, overdue, blocked, and categorized issue activity."
      },
      {
        "title": "REST API endpoints for pull requests",
        "url": "https://docs.github.com/en/rest/pulls/pulls",
        "used_for": "Recommendation to retrieve stale pull requests and pull requests awaiting review."
      },
      {
        "title": "About Projects",
        "url": "https://docs.github.com/en/issues/planning-and-tracking-with-projects/learning-about-projects/about-projects",
        "used_for": "Recommendation to use GitHub Projects fields and views for structured grouping and filtering."
      }
    ]
  },
  {
    "request": "For every new pull request, run the relevant Python tests and linters, compare coverage with the target branch, summarize failures in plain language, and suggest focused follow-up tasks without modifying unrelated files.",
    "options": [
      {
        "rank": 1,
        "name": "Custom GitHub Actions pull-request workflow",
        "reason": "Best fit: trigger on pull requests, run targeted Python tests and linters, calculate coverage against the base branch, and publish plain-language summaries and focused follow-up suggestions in the workflow summary or PR comment."
      },
      {
        "rank": 2,
        "name": "Reusable GitHub Actions quality workflow",
        "reason": "Encapsulates testing, linting, coverage comparison, and reporting into a reusable workflow that can be applied consistently across multiple Python repositories while limiting changes to relevant files."
      },
      {
        "rank": 3,
        "name": "GitHub Actions with Copilot-assisted PR reporting",
        "reason": "Combines deterministic Actions checks with Copilot-generated explanations and follow-up tasks, providing readable failure summaries without allowing automated changes to unrelated files."
      }
    ],
    "documentation_pages": []
  },
  {
    "request": "At the end of each quarter, analyze repository activity and maintenance trends, including contributor participation, issue resolution time, release frequency, dependency age, test health, and documentation gaps, then prepare an executive-friendly report with recommended priorities.",
    "options": [
      {
        "rank": 1,
        "name": "Scheduled GitHub Actions reporting workflow",
        "reason": "Best fit for repeatable quarterly automation: collect metrics through GitHub APIs and repository data, evaluate dependencies and tests, and publish an executive-friendly Markdown or artifact report with prioritized recommendations."
      },
      {
        "rank": 2,
        "name": "GitHub Insights plus a quarterly reporting template",
        "reason": "Uses native repository activity visualizations for contributors, issues, pull requests, and releases, supplemented with manually gathered dependency, test, and documentation findings."
      },
      {
        "rank": 3,
        "name": "GitHub CLI/API script run on demand",
        "reason": "A lightweight, customizable approach for maintainers who want to generate quarterly reports locally or from an operations runbook without maintaining a scheduled workflow."
      }
    ],
    "documentation_pages": [
      {
        "title": "Events that trigger workflows",
        "url": "https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule",
        "used_for": "Recommendation to run the quarterly analysis automatically with a scheduled GitHub Actions workflow."
      },
      {
        "title": "Using the REST API",
        "url": "https://docs.github.com/en/rest/using-the-rest-api/using-the-rest-api",
        "used_for": "Recommendation to collect repository, contributor, issue, pull request, release, and activity metrics programmatically."
      },
      {
        "title": "Viewing insights for your repository",
        "url": "https://docs.github.com/en/repositories/viewing-activity-and-data-for-your-repository/viewing-insights-for-your-repository",
        "used_for": "Recommendation to use native GitHub repository insights for activity, contributor, issue, pull request, and release trends."
      },
      {
        "title": "About the dependency graph",
        "url": "https://docs.github.com/en/code-security/concepts/supply-chain-security/understanding-your-software-supply-chain",
        "used_for": "Recommendation to incorporate dependency inventory and dependency-age analysis."
      }
    ]
  }
]