Skip to content

Commit 06eb889

Browse files
devin-ai-integration[bot]devin-ai-integration[bot]
authored
ci: daily upstream compatibility check for the framework integrations (SDK-22) (#347)
Daily at 06:41 UTC, each entry in .github/upstream-compat.json (fastmcp, mcp, temporalio, and the openai-agents extra to start) is typechecked and tested against its upstream's latest stable PyPI release. Floor-only ranges never need a PR: green records and red opens an issue saying users installing today are affected. A capped range gets a fix(keycardai-<package>) PR when a green version lands above the cap, flagged as breaking when it moves to a new major. State lives on the upstream-compat-state branch, merged by key with push retry.
1 parent aad625f commit 06eb889

4 files changed

Lines changed: 1084 additions & 0 deletions

File tree

‎.github/upstream-compat.json‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"entries": [
3+
{
4+
"package": "fastmcp",
5+
"upstream": "fastmcp",
6+
"field": "project.dependencies",
7+
"companions": []
8+
},
9+
{
10+
"package": "mcp",
11+
"upstream": "mcp",
12+
"field": "project.dependencies",
13+
"companions": []
14+
},
15+
{
16+
"package": "temporal",
17+
"upstream": "temporalio",
18+
"field": "project.dependencies",
19+
"companions": []
20+
},
21+
{
22+
"package": "temporal",
23+
"upstream": "openai-agents",
24+
"field": "project.optional-dependencies.openai-agents",
25+
"companions": []
26+
}
27+
]
28+
}

‎.github/workflows/upstream-compat.yml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
name: Upstream compatibility
2+
3+
# Tests each framework integration against its upstream's latest release
4+
# (SDK-22), mirroring typescript-sdk's workflow. Pairs live in
5+
# .github/upstream-compat.json; the logic in scripts/upstream_compat.py opens
6+
# the PR that supports a new version or the issue that records what it broke.
7+
8+
on:
9+
schedule:
10+
- cron: "41 6 * * *"
11+
workflow_dispatch:
12+
inputs:
13+
entry:
14+
description: "Config entry (<package>:<upstream>, or a package with one entry), or all"
15+
required: false
16+
default: all
17+
version:
18+
description: "Upstream version to test, or latest"
19+
required: false
20+
default: latest
21+
dry_run:
22+
description: "Print the PR or issue instead of opening it"
23+
required: false
24+
type: boolean
25+
default: false
26+
27+
jobs:
28+
entries:
29+
runs-on: ubuntu-latest
30+
outputs:
31+
matrix: ${{ steps.pick.outputs.matrix }}
32+
steps:
33+
- uses: actions/checkout@v4
34+
- id: pick
35+
env:
36+
ENTRY: ${{ inputs.entry || 'all' }}
37+
run: |
38+
matrix=$(python3 -c '
39+
import json, os
40+
entries = [f"{e[\"package\"]}:{e[\"upstream\"]}" for e in json.load(open(".github/upstream-compat.json"))["entries"]]
41+
pick = os.environ["ENTRY"]
42+
print(json.dumps(entries if pick == "all" else [pick]))
43+
')
44+
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
45+
46+
check:
47+
needs: entries
48+
runs-on: ubuntu-latest
49+
name: upstream (${{ matrix.entry }})
50+
strategy:
51+
fail-fast: false
52+
matrix:
53+
entry: ${{ fromJSON(needs.entries.outputs.matrix) }}
54+
permissions:
55+
contents: write
56+
pull-requests: write
57+
issues: write
58+
steps:
59+
# The release app's token, as in bump-package.yml: a PR opened with the
60+
# default GITHUB_TOKEN triggers no CI, and the support PR needs pr.yml.
61+
- name: Generate GitHub App token
62+
id: generate-token
63+
uses: actions/create-github-app-token@v2
64+
with:
65+
app-id: ${{ vars.SDK_RELEASE_APP_ID }}
66+
private-key: ${{ secrets.SDK_RELEASE_PRIVATE_KEY }}
67+
68+
- uses: actions/checkout@v4
69+
with:
70+
token: ${{ steps.generate-token.outputs.token }}
71+
fetch-depth: 0
72+
73+
- uses: actions/setup-python@v5
74+
with:
75+
python-version: "3.12"
76+
77+
- uses: astral-sh/setup-uv@v4
78+
79+
- name: Check ${{ matrix.entry }} against its upstream
80+
env:
81+
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
82+
ENTRY: ${{ matrix.entry }}
83+
VERSION: ${{ inputs.version || 'latest' }}
84+
DRY_RUN: ${{ inputs.dry_run && 'true' || 'false' }}
85+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
86+
run: |
87+
git config user.name "keycard-sdk-release[bot]"
88+
git config user.email "keycard-sdk-release[bot]@users.noreply.github.com"
89+
uv run --frozen python scripts/upstream_compat.py

‎scripts/test_upstream_compat.py‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
"""Unit tests for scripts/upstream_compat.py (SDK-22)."""
2+
3+
from __future__ import annotations
4+
5+
import unittest
6+
7+
import upstream_compat as uc
8+
9+
FASTMCP = uc.Entry("fastmcp", "fastmcp", "project.dependencies")
10+
MCP = uc.Entry("mcp", "mcp", "project.dependencies")
11+
AGENTS = uc.Entry(
12+
"temporal", "openai-agents", "project.optional-dependencies.openai-agents"
13+
)
14+
TEMPORALIO = uc.Entry("temporal", "temporalio", "project.dependencies")
15+
16+
PYPROJECT = """[project]
17+
name = "keycardai-temporal"
18+
dependencies = [
19+
"keycardai-oauth>=0.33.0",
20+
"temporalio>=1.32.0",
21+
]
22+
23+
[project.optional-dependencies]
24+
openai-agents = [
25+
"openai-agents>=0.20.0",
26+
"temporalio[openai-agents]>=1.32.0",
27+
]
28+
test = [
29+
"pytest>=8.0",
30+
]
31+
32+
[project.urls]
33+
Homepage = "https://keycard.ai"
34+
"""
35+
36+
37+
class SpecifierTests(unittest.TestCase):
38+
def test_open_floor_covers_every_later_release(self) -> None:
39+
self.assertTrue(uc.in_range(">=3.1.0", "4.1.0"))
40+
self.assertFalse(uc.has_cap(">=3.1.0"))
41+
self.assertEqual(uc.widened_range(">=3.1.0", "4.1.0"), ">=3.1.0")
42+
self.assertEqual(uc.decide(True, uc.in_range(">=3.1.0", "4.1.0")), "record")
43+
44+
def test_capped_same_major_moves_the_cap(self) -> None:
45+
self.assertTrue(uc.in_range(">=2.0.0,<3.0", "2.3.0"))
46+
self.assertFalse(uc.in_range(">=1.2.0,<1.9.0", "1.9.3"))
47+
self.assertEqual(uc.widened_range(">=1.2.0,<1.9.0", "1.9.3"), ">=1.2.0,<2.0")
48+
self.assertFalse(uc.is_new_major(">=1.2.0,<1.9.0", "1.9.3"))
49+
50+
def test_capped_new_major_starts_a_new_line(self) -> None:
51+
self.assertFalse(uc.in_range(">=2.0.0,<3.0", "3.1.0"))
52+
self.assertEqual(uc.widened_range(">=2.0.0,<3.0", "3.1.0"), ">=3.1.0,<4.0")
53+
self.assertTrue(uc.is_new_major(">=2.0.0,<3.0", "3.1.0"))
54+
self.assertIn(
55+
"breaking", uc.pr_body(MCP, "3.1.0", ">=2.0.0,<3.0", ">=3.1.0,<4.0", "")
56+
)
57+
self.assertNotIn(
58+
"breaking", uc.pr_body(MCP, "1.9.3", ">=1.2.0,<1.9.0", ">=1.2.0,<2.0", "")
59+
)
60+
61+
def test_zero_x_treats_each_minor_as_breaking(self) -> None:
62+
self.assertFalse(uc.in_range(">=0.20.0,<0.21.0", "0.23.1"))
63+
self.assertEqual(
64+
uc.widened_range(">=0.20.0,<0.21.0", "0.23.1"), ">=0.23.1,<0.24.0"
65+
)
66+
self.assertTrue(uc.is_new_major(">=0.20.0,<0.21.0", "0.23.1"))
67+
self.assertEqual(
68+
uc.widened_range(">=0.23.0,<0.23.1", "0.23.2"), ">=0.23.0,<0.24.0"
69+
)
70+
self.assertTrue(uc.in_range(">=0.20.0", "0.23.1"))
71+
72+
def test_other_comparators_are_kept(self) -> None:
73+
self.assertEqual(
74+
uc.widened_range(">=2.0.0,!=2.1.0,<3.0", "3.0.2"), ">=3.0.2,<4.0,!=2.1.0"
75+
)
76+
77+
def test_prereleases_and_yanked_files_are_not_latest(self) -> None:
78+
releases = {
79+
"4.1.0": [{"yanked": False}],
80+
"4.2.0rc1": [{"yanked": False}],
81+
"4.2.0.dev1": [{"yanked": False}],
82+
"4.1.1": [{"yanked": True}],
83+
"3.9.0": [{"yanked": False}],
84+
"garbage": [{"yanked": False}],
85+
}
86+
self.assertEqual(uc.latest_stable(releases), "4.1.0")
87+
88+
def test_decision_matrix(self) -> None:
89+
self.assertEqual(uc.decide(True, True), "record")
90+
self.assertEqual(uc.decide(True, False), "pr")
91+
self.assertEqual(uc.decide(False, True), "issue")
92+
self.assertEqual(uc.decide(False, False), "issue")
93+
94+
def test_issue_body_says_whether_users_are_affected(self) -> None:
95+
inside = uc.issue_body(FASTMCP, "4.1.0", ">=3.1.0", True, "E boom", "")
96+
self.assertIn(
97+
"users installing keycardai-fastmcp today already get this", inside
98+
)
99+
outside = uc.issue_body(MCP, "3.1.0", ">=2.0.0,<3.0", False, "E boom", "")
100+
self.assertIn("users are not affected until the specifier widens", outside)
101+
102+
103+
class PyprojectTests(unittest.TestCase):
104+
def test_reads_dependencies_and_extras(self) -> None:
105+
self.assertEqual(uc.read_declared(PYPROJECT, "temporalio", None), ">=1.32.0")
106+
self.assertEqual(
107+
uc.read_declared(PYPROJECT, "openai-agents", "openai-agents"), ">=0.20.0"
108+
)
109+
self.assertEqual(AGENTS.extra, "openai-agents")
110+
self.assertIsNone(TEMPORALIO.extra)
111+
112+
def test_replace_touches_only_the_named_declaration(self) -> None:
113+
out = uc.replace_declared(
114+
PYPROJECT, "openai-agents", "openai-agents", ">=0.23.1,<0.24.0"
115+
)
116+
self.assertIn('"openai-agents>=0.23.1,<0.24.0",', out)
117+
self.assertIn('"temporalio[openai-agents]>=1.32.0",', out)
118+
self.assertIn('"temporalio>=1.32.0",', out)
119+
self.assertEqual(len(out.splitlines()), len(PYPROJECT.splitlines()))
120+
out2 = uc.replace_declared(PYPROJECT, "temporalio", None, ">=1.34.0")
121+
self.assertIn('"temporalio>=1.34.0",', out2)
122+
self.assertIn('"temporalio[openai-agents]>=1.32.0",', out2)
123+
124+
125+
class DedupeTests(unittest.TestCase):
126+
def test_names(self) -> None:
127+
self.assertEqual(
128+
uc.branch_name(AGENTS, "0.23.1"), "upstream/temporal-openai-agents-0.23.1"
129+
)
130+
self.assertEqual(
131+
uc.pr_title(MCP, "3.1.0"), "fix(keycardai-mcp): support mcp 3.1.0"
132+
)
133+
self.assertEqual(
134+
uc.issue_title(FASTMCP, "4.1.0"), "fastmcp 4.1.0 breaks keycardai-fastmcp"
135+
)
136+
self.assertEqual(AGENTS.key, "temporal:openai-agents")
137+
138+
def test_pick_entry_by_key_or_unique_package(self) -> None:
139+
entries = [FASTMCP, MCP, TEMPORALIO, AGENTS]
140+
self.assertIs(uc.pick_entry(entries, "fastmcp"), FASTMCP)
141+
self.assertIs(uc.pick_entry(entries, "temporal:openai-agents"), AGENTS)
142+
with self.assertRaises(SystemExit):
143+
uc.pick_entry(entries, "temporal")
144+
with self.assertRaises(SystemExit):
145+
uc.pick_entry(entries, "langchain")
146+
147+
def test_exact_title_match_only(self) -> None:
148+
items = [
149+
{"number": 7, "title": "fastmcp 4.1.0 breaks keycardai-fastmcp"},
150+
{"number": 8, "title": "fastmcp 4.1.1 breaks keycardai-fastmcp"},
151+
]
152+
self.assertEqual(
153+
uc.find_by_title(items, "fastmcp 4.1.0 breaks keycardai-fastmcp"), 7
154+
)
155+
self.assertIsNone(
156+
uc.find_by_title(items, "fastmcp 4.2.0 breaks keycardai-fastmcp")
157+
)
158+
159+
def test_failed_differently(self) -> None:
160+
body = uc.issue_body(
161+
FASTMCP, "4.1.0", ">=3.1.0", True, "E ImportError: x", ""
162+
)
163+
self.assertFalse(uc.failed_differently(body, "E ImportError: x"))
164+
self.assertTrue(uc.failed_differently(body, "E TypeError: y"))
165+
166+
def test_already_tested(self) -> None:
167+
state = {"fastmcp:fastmcp": {"version": "4.1.0", "run": "r"}}
168+
self.assertTrue(uc.already_tested(state, FASTMCP, "4.1.0"))
169+
self.assertFalse(uc.already_tested(state, FASTMCP, "4.1.1"))
170+
self.assertFalse(uc.already_tested(state, MCP, "4.1.0"))
171+
172+
173+
class StateWriteTests(unittest.TestCase):
174+
def test_merge_by_key_keeps_the_other_entries(self) -> None:
175+
start = {"mcp:mcp": {"version": "2.3.0", "run": "r1"}}
176+
a = uc.merge_state(start, "fastmcp:fastmcp", {"version": "4.1.0", "run": "r2"})
177+
b = uc.merge_state(
178+
start, "temporal:temporalio", {"version": "1.34.0", "run": "r3"}
179+
)
180+
self.assertEqual(set(a), {"mcp:mcp", "fastmcp:fastmcp"})
181+
self.assertEqual(set(b), {"mcp:mcp", "temporal:temporalio"})
182+
self.assertEqual(start, {"mcp:mcp": {"version": "2.3.0", "run": "r1"}})
183+
184+
def test_rejected_push_rereads_the_tip_and_retries_once(self) -> None:
185+
tips = iter([{}, {"mcp:mcp": {"version": "2.3.0", "run": "r1"}}])
186+
results = iter([False, True])
187+
writes: list[dict] = []
188+
sleeps: list[float] = []
189+
190+
def write_once(repo_root: str, branch: str, state: dict, message: str) -> bool:
191+
writes.append(state)
192+
return next(results)
193+
194+
final = uc.record_state(
195+
"/repo",
196+
"upstream-compat-state",
197+
"fastmcp:fastmcp",
198+
{"version": "4.1.0", "run": "r2"},
199+
"msg",
200+
read=lambda root, branch: next(tips),
201+
write_once=write_once,
202+
sleep=sleeps.append,
203+
)
204+
self.assertEqual(len(writes), 2)
205+
self.assertEqual(set(final), {"mcp:mcp", "fastmcp:fastmcp"})
206+
self.assertEqual(len(sleeps), 1)
207+
208+
def test_exhausted_attempts_fail_as_a_state_write(self) -> None:
209+
with self.assertRaises(SystemExit) as ctx:
210+
uc.record_state(
211+
"/repo",
212+
"upstream-compat-state",
213+
"mcp:mcp",
214+
{"version": "2.3.0", "run": ""},
215+
"msg",
216+
attempts=3,
217+
read=lambda root, branch: {},
218+
write_once=lambda *a: False,
219+
sleep=lambda s: None,
220+
)
221+
self.assertIn("state write failed after 3 attempts", str(ctx.exception))
222+
223+
224+
if __name__ == "__main__":
225+
unittest.main()

0 commit comments

Comments
 (0)