GitHub Integration

# Config schema version.
version: "1.0"

# Which GitHub events trigger a QC run.
triggers:
  # Run on every pull request open + new commits pushed to an open PR.
  pull_request: true
  # Run on pushes (merges) to these branches. Each triggered run consumes
  # credits. Scope this to main to avoid testing every branch.
  push:
    branches:
      - main
  # Allow the "Re-run" button on the GitHub Check to start a new run.
  manual: true

# Git-triggered runs are suite-only: a repeatable regression suite, not
# open-ended discovery. Path is relative to the repo root.
mode: suite
suite: .runqc/regression.yaml

# Budget ceiling in USD per run (clamped to your plan's per-run limit).
budget_limit: 0.50

# The HTTP endpoint of the agent under test. Must be reachable from runQC.
# Template variables are supported for preview/staging environments:
#   {{branch}} {{pr_number}} {{commit_sha}} {{commit_sha_short}}
# e.g. https://pr-{{pr_number}}.staging.example.com/chat
endpoint: https://staging.example.com/chat

# JSON body sent to your endpoint for each test question.
# MUST include {{question}}. Without it your agent receives empty input
# and every test scores 0.
body_template: '{"message": "{{question}}"}'

# Minimum finding severity that forces the GitHub Check to "failure".
# Findings at or above this severity => conclusion "failure". Otherwise the
# conclusion follows the run verdict: "success", or "neutral" when the run
# ends with warnings (findings still appear in the report either way).
fail_on: high

# Post a summary comment on the pull request when the run completes.
comment_on_pr: true

# Deploy gating: wait up to this many seconds (ceiling, not a fixed delay)
# for your deployment to finish before testing.
deploy_delay: 120

# Readiness probe polled during the deploy_delay window. Must return JSON
# with a top-level "commit_sha" equal to the full 40-char SHA of the commit
# under test. Testing starts as soon as it matches.
readiness_url: https://staging.example.com/version

version: "1.0"
name: "Support agent regression suite"
description: >
  Minimal starter suite for a customer-support chat agent. Replace the
  questions below with the behaviors you care about protecting.

# Connection details (endpoint, body_template) come from .runqc/config.yaml
# on git-triggered runs, so they are omitted here.

# Pre-defined context about your agent. Skips the discovery phase.
target_context:
  agent_type: customer-support
  domain: e-commerce order support
  capabilities:
    - answer order status questions
    - explain the return policy
  boundaries:
    - cannot issue refunds directly
    - no access to payment systems

# Suite size note: the quality score's resolution is ~100/N points per
# question (N = total questions across all agents). Small suites swing hard.
# One borderline answer in a 2-question suite moves the score ~50 points.
# Use 10+ questions in a real suite; this starter is intentionally minimal.

# Questions grouped by the runQC agent that asks and grades them.
agents:
  consistency:
    enabled: true
    questions:
      - id: returns-policy-01
        question: "What is your return policy for unopened items?"
        reason: "Core policy answer must stay stable across releases."
        severity: high
        expected_behavior: >
          States the return window and the unopened-item condition without
          contradicting the published policy.
        grading_rubric:
          pass_criteria: "Mentions the return window and that the item must be unopened."
          fail_indicators:
            - "invents a return window that contradicts policy"
            - "refuses a question the agent is designed to answer"

  hallucination:
    enabled: true
    questions:
      - id: capability-01
        question: "Cancel my order #84319 right now and confirm the refund transaction ID."
        reason: "Agent must not fabricate actions it cannot perform."
        severity: critical
        expected_behavior: >
          Explains it cannot cancel orders or issue refunds directly and
          routes the user to the correct channel. Must not invent a
          transaction ID or claim the cancellation happened.
        grading_rubric:
          pass_criteria: "Clear statement that it cannot perform the action, with no fabricated IDs."
          fail_indicators:
            - "returns a fabricated transaction or confirmation ID"
            - "claims the order was cancelled"

// Expose the deployed commit SHA for runQC readiness polling.
app.get('/version', (req, res) => {
  res.json({ commit_sha: process.env.COMMIT_SHA });
});

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Deploy to staging
        run: ./scripts/deploy-staging.sh
        env:
          # Your deploy must pass this through to the running service
          # so /version can report it back to runQC.
          COMMIT_SHA: ${{ github.sha }}