Skip to content

Commit ab0e35a

Browse files
ashwinbashwinb
committed
feat: add integration tests CI workflow
Add GitHub Actions workflow to run integration tests against the llama-stack server in replay mode. The workflow dynamically generates a test matrix from llama-stack's CI configuration and runs TypeScript client tests alongside Python tests. Key features: - Leverages llama-stack's existing test infrastructure and recorded API responses - Only runs for suite/setup combinations that have TypeScript test coverage - Filters matrix to server mode only (TypeScript client doesn't support library/docker modes) - Builds local client and installs it in llama-stack's test environment via TS_CLIENT_PATH Triggers on pushes/PRs to main and generated branches, plus manual dispatch. Test plan: Will be validated when workflow runs in CI.
1 parent e78b187 commit ab0e35a

File tree

1 file changed

+208
-0
lines changed

1 file changed

+208
-0
lines changed

‎.github/workflows/integration-tests.yml‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
name: Integration Tests (Client-side)
2+
3+
run-name: Run integration tests from llama-stack using local TypeScript client
4+
5+
on:
6+
push:
7+
branches:
8+
- main
9+
- generated
10+
- 'release-[0-9]+.[0-9]+.x'
11+
pull_request:
12+
branches:
13+
- main
14+
- generated
15+
- 'release-[0-9]+.[0-9]+.x'
16+
types: [opened, synchronize, reopened]
17+
paths:
18+
- 'src/**'
19+
- 'tests/integration/**'
20+
- 'package.json'
21+
- 'yarn.lock'
22+
- '.github/workflows/integration-tests.yml'
23+
workflow_dispatch:
24+
inputs:
25+
test-setup:
26+
description: 'Test against a specific setup'
27+
type: string
28+
default: 'ollama'
29+
30+
concurrency:
31+
# Skip concurrency for pushes to main - each commit should be tested independently
32+
group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
33+
cancel-in-progress: true
34+
35+
jobs:
36+
generate-matrix:
37+
runs-on: ubuntu-latest
38+
outputs:
39+
matrix: ${{ steps.set-matrix.outputs.matrix }}
40+
steps:
41+
- name: Checkout llama-stack repository
42+
uses: actions/checkout@v4
43+
with:
44+
repository: llamastack/llama-stack
45+
ref: main
46+
47+
- name: Generate test matrix
48+
id: set-matrix
49+
run: |
50+
# Generate matrix from CI_MATRIX in llama-stack tests/integration/ci_matrix.json
51+
MATRIX=$(PYTHONPATH=. python3 scripts/generate_ci_matrix.py \
52+
--test-setup "${{ github.event.inputs.test-setup }}")
53+
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
54+
echo "Generated matrix: $MATRIX"
55+
56+
run-replay-mode-tests:
57+
needs: generate-matrix
58+
runs-on: ubuntu-latest
59+
name: ${{ format('Integration Tests (server, {0}, {1})', matrix.config.setup, matrix.config.suite) }}
60+
61+
strategy:
62+
fail-fast: false
63+
matrix:
64+
# Only test server mode - TypeScript client doesn't support library/docker modes
65+
client: [server]
66+
# Test configurations: Generated from CI_MATRIX in llama-stack tests/integration/ci_matrix.json
67+
config: ${{ fromJSON(needs.generate-matrix.outputs.matrix).include }}
68+
69+
steps:
70+
- name: Checkout llama-stack-client-typescript repository
71+
uses: actions/checkout@v4
72+
with:
73+
path: llama-stack-client-typescript
74+
75+
- name: Checkout llama-stack repository
76+
uses: actions/checkout@v4
77+
with:
78+
repository: llamastack/llama-stack
79+
ref: main
80+
path: llama-stack
81+
82+
- name: Check if TypeScript tests exist for this config
83+
id: check-ts-tests
84+
working-directory: llama-stack
85+
run: |
86+
# Check if this config allows server client (TypeScript only works with server mode)
87+
ALLOWED_CLIENTS='${{ toJSON(matrix.config.allowed_clients) }}'
88+
if [[ "$ALLOWED_CLIENTS" != "null" ]] && ! echo "$ALLOWED_CLIENTS" | grep -q "server"; then
89+
echo "has_tests=false" >> $GITHUB_OUTPUT
90+
echo "⚠️ Config doesn't allow server client, skipping TypeScript tests"
91+
exit 0
92+
fi
93+
94+
# Check if this suite/setup combination has TypeScript tests
95+
SUITE="${{ matrix.config.suite }}"
96+
SETUP="${{ matrix.config.setup }}"
97+
98+
if [[ ! -f "tests/integration/client-typescript/suites.json" ]]; then
99+
echo "has_tests=false" >> $GITHUB_OUTPUT
100+
echo "No TypeScript test mapping found"
101+
exit 0
102+
fi
103+
104+
# Check if suite/setup is mapped in suites.json
105+
HAS_TESTS=$(python3 -c "
106+
import json
107+
with open('tests/integration/client-typescript/suites.json') as f:
108+
suites = json.load(f)
109+
for entry in suites:
110+
if entry.get('suite') == '$SUITE' and entry.get('setup') == '$SETUP':
111+
print('true')
112+
exit(0)
113+
print('false')
114+
")
115+
116+
echo "has_tests=$HAS_TESTS" >> $GITHUB_OUTPUT
117+
if [[ "$HAS_TESTS" == "true" ]]; then
118+
echo "✅ TypeScript tests found for suite=$SUITE setup=$SETUP"
119+
else
120+
echo "⚠️ No TypeScript tests mapped for suite=$SUITE setup=$SETUP"
121+
fi
122+
123+
- name: Set up Node.js
124+
if: steps.check-ts-tests.outputs.has_tests == 'true'
125+
uses: actions/setup-node@v4
126+
with:
127+
node-version: '20'
128+
129+
- name: Install uv
130+
if: steps.check-ts-tests.outputs.has_tests == 'true'
131+
uses: astral-sh/setup-uv@v6
132+
with:
133+
python-version: '3.12'
134+
version: '0.7.6'
135+
136+
- name: Build TypeScript client
137+
if: steps.check-ts-tests.outputs.has_tests == 'true'
138+
working-directory: llama-stack-client-typescript
139+
run: |
140+
echo "Building TypeScript client..."
141+
yarn install --frozen-lockfile
142+
yarn build
143+
144+
echo "✅ TypeScript client built successfully"
145+
146+
- name: Install llama-stack dependencies
147+
if: steps.check-ts-tests.outputs.has_tests == 'true'
148+
working-directory: llama-stack
149+
run: |
150+
echo "Installing llama-stack dependencies"
151+
uv sync --all-groups
152+
uv pip install faiss-cpu
153+
154+
- name: Build Llama Stack
155+
if: steps.check-ts-tests.outputs.has_tests == 'true'
156+
working-directory: llama-stack
157+
run: |
158+
echo "Building Llama Stack"
159+
LLAMA_STACK_DIR=. \
160+
uv run --no-sync llama stack list-deps ci-tests | xargs -L1 uv pip install
161+
162+
- name: Configure git for commits
163+
if: steps.check-ts-tests.outputs.has_tests == 'true'
164+
working-directory: llama-stack
165+
run: |
166+
git config --local user.email "github-actions[bot]@users.noreply.github.com"
167+
git config --local user.name "github-actions[bot]"
168+
169+
- name: Check Storage and Memory Available Before Tests
170+
if: steps.check-ts-tests.outputs.has_tests == 'true' && always()
171+
run: |
172+
free -h
173+
df -h
174+
175+
- name: Run Integration Tests (Replay Mode)
176+
if: steps.check-ts-tests.outputs.has_tests == 'true'
177+
working-directory: llama-stack
178+
env:
179+
OPENAI_API_KEY: dummy
180+
TS_CLIENT_PATH: ${{ github.workspace }}/llama-stack-client-typescript
181+
run: |
182+
STACK_CONFIG="${{ matrix.config.stack_config || 'server:ci-tests' }}"
183+
184+
SCRIPT_ARGS="--stack-config $STACK_CONFIG --inference-mode replay"
185+
186+
# Add optional arguments
187+
if [ -n '${{ matrix.config.setup }}' ]; then
188+
SCRIPT_ARGS="$SCRIPT_ARGS --setup ${{ matrix.config.setup }}"
189+
fi
190+
if [ -n '${{ matrix.config.suite }}' ]; then
191+
SCRIPT_ARGS="$SCRIPT_ARGS --suite ${{ matrix.config.suite }}"
192+
fi
193+
194+
echo "=== Running integration tests with TypeScript client ==="
195+
echo "Client path: $TS_CLIENT_PATH"
196+
echo "Command: uv run --no-sync ./scripts/integration-tests.sh $SCRIPT_ARGS"
197+
echo ""
198+
199+
uv run --no-sync ./scripts/integration-tests.sh $SCRIPT_ARGS | tee pytest-replay.log
200+
201+
- name: Upload logs
202+
if: always()
203+
uses: actions/upload-artifact@v4
204+
with:
205+
name: logs-${{ github.run_id }}-${{ github.run_attempt || '1' }}-${{ strategy.job-index || github.job }}-${{ github.action }}
206+
path: |
207+
llama-stack/*.log
208+
retention-days: 1

0 commit comments

Comments
 (0)