Skip to content

Commit 6ba923b

Browse files
authored
ci: post coverage delta comment on PRs (#829)
Similar was implemented in a2aproject/a2a-js#270. Tested on fork: <img width="910" height="651" alt="Screenshot 2026-03-13 at 14 31 36" src="https://github.com/user-attachments/assets/374c6bd1-0437-419d-b1f3-409b55001fe2" /> <img width="885" height="625" alt="Screenshot 2026-03-13 at 14 32 09" src="https://github.com/user-attachments/assets/9a12fa03-35c4-4c44-814e-87312598a0b5" />
1 parent 63f4a28 commit 6ba923b

2 files changed

Lines changed: 250 additions & 2 deletions

File tree

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
name: Post Coverage Comment
2+
3+
on:
4+
workflow_run:
5+
workflows: ["Run Unit Tests"]
6+
types:
7+
- completed
8+
9+
permissions:
10+
pull-requests: write
11+
actions: read
12+
13+
jobs:
14+
comment:
15+
runs-on: ubuntu-latest
16+
if: >
17+
github.event.workflow_run.event == 'pull_request' &&
18+
github.event.workflow_run.conclusion == 'success'
19+
steps:
20+
- name: Download Coverage Artifacts
21+
uses: actions/download-artifact@v4
22+
with:
23+
run-id: ${{ github.event.workflow_run.id }}
24+
github-token: ${{ secrets.A2A_BOT_PAT }}
25+
name: coverage-data
26+
27+
- name: Upload Coverage Report
28+
id: upload-report
29+
uses: actions/upload-artifact@v4
30+
with:
31+
name: coverage-report
32+
path: coverage/
33+
retention-days: 14
34+
35+
- name: Post Comment
36+
uses: actions/github-script@v6
37+
env:
38+
ARTIFACT_URL: ${{ steps.upload-report.outputs.artifact-url }}
39+
with:
40+
script: |
41+
const fs = require('fs');
42+
43+
const { owner, repo } = context.repo;
44+
const headSha = context.payload.workflow_run.head_commit.id;
45+
46+
const loadSummary = (path) => {
47+
try {
48+
const data = JSON.parse(fs.readFileSync(path, 'utf8'));
49+
// Map Python coverage.json format to expected internal summary format
50+
if (data.totals && data.files) {
51+
const summary = {
52+
total: {
53+
statements: { pct: data.totals.percent_covered }
54+
}
55+
};
56+
for (const [file, fileData] of Object.entries(data.files)) {
57+
// Python coverage uses absolute paths or relative to project root
58+
// We keep it as is for comparison
59+
summary[file] = {
60+
statements: { pct: fileData.summary.percent_covered }
61+
};
62+
}
63+
return summary;
64+
}
65+
return data;
66+
} catch (e) {
67+
console.log(`Could not read ${path}: ${e}`);
68+
return null;
69+
}
70+
};
71+
72+
const baseSummary = loadSummary('./coverage-base.json');
73+
const prSummary = loadSummary('./coverage-pr.json');
74+
75+
if (!baseSummary || !prSummary) {
76+
console.log("Missing coverage data, skipping comment.");
77+
return;
78+
}
79+
80+
let baseBranch = 'main';
81+
try {
82+
baseBranch = fs.readFileSync('./BASE_BRANCH', 'utf8').trim();
83+
} catch (e) {
84+
console.log("Could not read BASE_BRANCH, defaulting to main.");
85+
}
86+
87+
let markdown = `### 🧪 Code Coverage (vs \`${baseBranch}\`)\n\n`;
88+
89+
markdown += `[⬇️ **Download Full Report**](${process.env.ARTIFACT_URL})\n\n`;
90+
91+
const metric = 'statements';
92+
const getPct = (summaryItem, m) => summaryItem && summaryItem[m] ? Number(summaryItem[m].pct) : 0;
93+
94+
const formatDiff = (oldPct, newPct) => {
95+
const diff = (newPct - oldPct).toFixed(2);
96+
97+
let icon = '';
98+
if (diff > 0) icon = '🟢';
99+
else if (diff < 0) icon = '🔴';
100+
else icon = '⚪️';
101+
102+
const diffStr = diff > 0 ? `+${diff}%` : `${diff}%`;
103+
return `${icon} ${diffStr}`;
104+
};
105+
106+
const fileUrl = (path) => `https://github.com/${owner}/${repo}/blob/${headSha}/${path}`;
107+
108+
const allFiles = new Set([...Object.keys(baseSummary), ...Object.keys(prSummary)]);
109+
allFiles.delete('total');
110+
const workspacePath = process.env.GITHUB_WORKSPACE ? process.env.GITHUB_WORKSPACE + '/' : '';
111+
112+
let changedRows = [];
113+
let newRows = [];
114+
115+
for (const file of allFiles) {
116+
const baseFile = baseSummary[file];
117+
const prFile = prSummary[file];
118+
119+
if (!prFile) continue;
120+
121+
const oldPct = getPct(baseFile, metric);
122+
const newPct = getPct(prFile, metric);
123+
124+
const relativeFilePath = file.replace(workspacePath, '');
125+
const linkedPath = `[${relativeFilePath}](${fileUrl(relativeFilePath)})`;
126+
127+
if (!baseFile && prFile) {
128+
newRows.push(`| ${linkedPath} (**new**) | — | ${newPct.toFixed(2)}% | — |\n`);
129+
} else if (oldPct !== newPct) {
130+
changedRows.push(`| ${linkedPath} | ${oldPct.toFixed(2)}% | ${newPct.toFixed(2)}% | ${formatDiff(oldPct, newPct)} |\n`);
131+
}
132+
}
133+
134+
if (changedRows.length === 0 && newRows.length === 0) {
135+
markdown += `\n_No coverage changes._\n`;
136+
} else {
137+
markdown += `| | Base | PR | Delta |\n`;
138+
markdown += `| :--- | :---: | :---: | :---: |\n`;
139+
140+
if (changedRows.length > 0) {
141+
markdown += changedRows.sort().join('');
142+
}
143+
if (newRows.length > 0) {
144+
markdown += newRows.sort().join('');
145+
}
146+
147+
const oldTotalPct = getPct(baseSummary.total, metric);
148+
const newTotalPct = getPct(prSummary.total, metric);
149+
if (oldTotalPct !== newTotalPct) {
150+
markdown += `| **Total** | ${oldTotalPct.toFixed(2)}% | ${newTotalPct.toFixed(2)}% | ${formatDiff(oldTotalPct, newTotalPct)} |\n`;
151+
}
152+
}
153+
154+
markdown += `\n\n_Generated by [coverage-comment.yml](https://github.com/${owner}/${repo}/actions/workflows/coverage-comment.yml)_`;
155+
156+
const prNumber = fs.readFileSync('./PR_NUMBER', 'utf8').trim();
157+
158+
if (!prNumber) {
159+
console.log("No PR number found.");
160+
return;
161+
}
162+
163+
const comments = await github.rest.issues.listComments({
164+
owner,
165+
repo,
166+
issue_number: prNumber,
167+
});
168+
169+
const existingComment = comments.data.find(c =>
170+
c.body.includes('Generated by [coverage-comment.yml]') &&
171+
c.user.type === 'Bot'
172+
);
173+
174+
if (existingComment) {
175+
await github.rest.issues.updateComment({
176+
owner,
177+
repo,
178+
comment_id: existingComment.id,
179+
body: markdown
180+
});
181+
} else {
182+
await github.rest.issues.createComment({
183+
owner,
184+
repo,
185+
issue_number: prNumber,
186+
body: markdown
187+
});
188+
}

.github/workflows/unit-tests.yml

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
---
22
name: Run Unit Tests
33
on:
4-
pull_request:
4+
push:
55
branches: [main]
6+
pull_request:
67
permissions:
78
contents: read
9+
810
jobs:
911
test:
1012
name: Test with Python ${{ matrix.python-version }}
@@ -54,7 +56,65 @@ jobs:
5456
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
5557
- name: Install dependencies
5658
run: uv sync --locked
57-
- name: Run tests and check coverage
59+
60+
# Coverage comparison for PRs (only on Python 3.13 to avoid duplicate work)
61+
- name: Checkout Base Branch
62+
if: github.event_name == 'pull_request' && matrix.python-version == '3.13'
63+
uses: actions/checkout@v4
64+
with:
65+
ref: ${{ github.event.pull_request.base.ref || 'main' }}
66+
clean: true
67+
68+
- name: Run coverage (Base)
69+
if: github.event_name == 'pull_request' && matrix.python-version == '3.13'
70+
run: |
71+
uv run pytest --cov=a2a --cov-report=json --cov-report=html:coverage
72+
mv coverage.json /tmp/coverage-base.json
73+
74+
- name: Checkout PR Branch (Restore)
75+
if: github.event_name == 'pull_request' && matrix.python-version == '3.13'
76+
uses: actions/checkout@v4
77+
with:
78+
clean: true
79+
80+
- name: Run coverage (PR)
81+
if: github.event_name == 'pull_request' && matrix.python-version == '3.13'
82+
run: |
83+
uv run pytest --cov=a2a --cov-report=json --cov-report=html:coverage --cov-report=term --cov-fail-under=88
84+
mv coverage.json coverage-pr.json
85+
cp /tmp/coverage-base.json coverage-base.json
86+
87+
- name: Save Metadata
88+
if: github.event_name == 'pull_request' && matrix.python-version == '3.13'
89+
run: |
90+
echo ${{ github.event.number }} > ./PR_NUMBER
91+
echo ${{ github.event.pull_request.base.ref || 'main' }} > ./BASE_BRANCH
92+
93+
- name: Upload Coverage Artifacts
94+
uses: actions/upload-artifact@v4
95+
if: github.event_name == 'pull_request' && matrix.python-version == '3.13'
96+
with:
97+
name: coverage-data
98+
path: |
99+
coverage-base.json
100+
coverage-pr.json
101+
coverage/
102+
PR_NUMBER
103+
BASE_BRANCH
104+
retention-days: 1
105+
106+
# Run standard tests (for matrix items that didn't run coverage PR)
107+
- name: Run tests (Standard)
108+
if: matrix.python-version != '3.13' || github.event_name != 'pull_request'
58109
run: uv run pytest --cov=a2a --cov-report term --cov-fail-under=88
110+
111+
- name: Upload Artifact (base)
112+
uses: actions/upload-artifact@v4
113+
if: github.event_name != 'pull_request' && matrix.python-version == '3.13'
114+
with:
115+
name: coverage-report
116+
path: coverage
117+
retention-days: 14
118+
59119
- name: Show coverage summary in log
60120
run: uv run coverage report

0 commit comments

Comments
 (0)