Skip to content

Commit 08ff3de

Browse files
mumrahmumrah
authored
add missing workflow (#49)
1 parent b102bf5 commit 08ff3de

File tree

1 file changed

+284
-0
lines changed

1 file changed

+284
-0
lines changed

‎.github/workflows/build.yml‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: Check and Test
17+
18+
# This workflow should only be called from ci.yml which is triggered on
19+
# the "pull_request" event type. We should never dispatch this workflow from
20+
# a "pull_request_target" event.
21+
on:
22+
workflow_call:
23+
inputs:
24+
is-trunk:
25+
description: "Is this a trunk build?"
26+
default: true
27+
type: boolean
28+
is-public-fork:
29+
description: "Is this CI run from a public fork?"
30+
default: true
31+
type: boolean
32+
33+
jobs:
34+
load-catalog:
35+
runs-on: ubuntu-latest
36+
name: Load Test Catalog
37+
steps:
38+
- name: Checkout main
39+
uses: actions/checkout@v4
40+
with:
41+
persist-credentials: false
42+
43+
- name: Checkout test-catalog
44+
uses: actions/checkout@v4
45+
with:
46+
ref: 'test-catalog'
47+
persist-credentials: false
48+
fetch-depth: 100 # Needs to be large enough to ensure we can fetch N days ago
49+
path: test-catalog
50+
51+
- name: Checkout catalog at earlier date
52+
run: |
53+
cd test-catalog
54+
SHA=$(git rev-list -1 --before 7.days.ago origin/test-catalog)
55+
echo $SHA
56+
git switch --detach $SHA
57+
git show --no-patch
58+
59+
- name: Setup Python
60+
uses: ./.github/actions/setup-python
61+
62+
# Prior to this step, we don't expect any errors. For the rest of this "load-catalog" job, we need to ensure
63+
# we do not fail as this will fail the overall workflow.
64+
- name: Combine Catalog into single file
65+
id: combine-catalog
66+
continue-on-error: true
67+
run: |
68+
python .github/scripts/format-test-catalog.py --path "test-catalog/test-catalog/**/*.yaml"
69+
70+
- name: Archive Combined Test Catalog
71+
if: steps.combine-catalog.outcome == 'success'
72+
uses: actions/upload-artifact@v4
73+
with:
74+
name: combined-test-catalog
75+
path: combined-test-catalog.txt
76+
compression-level: 9
77+
78+
validate:
79+
runs-on: ubuntu-latest
80+
name: Compile and Check Java
81+
outputs:
82+
is-draft: ${{ steps.check-draft-pr.outputs.is-draft }}
83+
steps:
84+
- name: Env
85+
run: printenv
86+
env:
87+
GITHUB_CONTEXT: ${{ toJson(github) }}
88+
- name: Check for Draft PR
89+
id: check-draft-pr
90+
if: |
91+
github.event_name == 'pull_request' &&
92+
github.event.pull_request.draft
93+
run: echo "is-draft=true" >> "$GITHUB_OUTPUT"
94+
- name: Checkout code
95+
uses: actions/checkout@v4
96+
with:
97+
persist-credentials: false
98+
- name: Setup Python
99+
uses: ./.github/actions/setup-python
100+
- name: Setup Gradle
101+
uses: ./.github/actions/setup-gradle
102+
with:
103+
java-version: 23
104+
gradle-cache-read-only: ${{ !inputs.is-trunk }}
105+
gradle-cache-write-only: ${{ inputs.is-trunk }}
106+
develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
107+
- name: Compile and validate
108+
env:
109+
SCAN_ARG: ${{ inputs.is-public-fork && '--no-scan' || '--scan' }}
110+
# Gradle flags
111+
# --build-cache: Let Gradle restore the build cache
112+
# --info: For now, we'll generate lots of logs while setting up the GH Actions
113+
# --scan: Publish the build scan. This will only work on PRs from apache/kafka and trunk
114+
# --no-scan: For public fork PRs, we won't attempt to publish the scan
115+
run: |
116+
./gradlew --build-cache --info $SCAN_ARG check siteDocTar -x test
117+
- name: Archive check reports
118+
if: always()
119+
uses: actions/upload-artifact@v4
120+
with:
121+
name: check-reports
122+
path: |
123+
**/build/**/*.html
124+
compression-level: 9
125+
if-no-files-found: ignore
126+
- name: Annotate checkstyle errors
127+
if: failure()
128+
run: python .github/scripts/checkstyle.py
129+
env:
130+
GITHUB_WORKSPACE: ${{ github.workspace }}
131+
- name: Annotate Rat errors
132+
if: failure()
133+
run: python .github/scripts/rat.py
134+
env:
135+
GITHUB_WORKSPACE: ${{ github.workspace }}
136+
- name: Check generated documentation
137+
# Check if there are any empty files under ./site-docs/generated, If any empty files are found, print an error
138+
# message and list the empty files
139+
run: |
140+
tar zxvf core/build/distributions/kafka_2.13-$(./gradlew properties | grep version: | awk '{print $NF}' | head -n 1)-site-docs.tgz
141+
if find ./site-docs/generated -type f -exec grep -L "." {} \; | grep -q "."; then
142+
echo "One or more documentation files are empty!" >&2
143+
find ./site-docs/generated -type f -exec grep -L "." {} \; >&2
144+
exit 1
145+
fi
146+
147+
test:
148+
needs: [validate, load-catalog]
149+
if: ${{ ! needs.validate.outputs.is-draft }}
150+
runs-on: ubuntu-latest
151+
strategy:
152+
fail-fast: false
153+
matrix:
154+
java: [ 23, 17 ] # If we change these, make sure to adjust ci-complete.yml
155+
outputs:
156+
timed-out: ${{ (steps.junit-test.outputs.gradle-exitcode == '124' || steps.junit-quarantined-test.outputs.gradle-exitcode == '124') }}
157+
name: JUnit tests Java ${{ matrix.java }}
158+
steps:
159+
- name: Checkout code
160+
uses: actions/checkout@v4
161+
with:
162+
persist-credentials: false
163+
- name: Setup Python
164+
uses: ./.github/actions/setup-python
165+
- name: Setup Gradle
166+
uses: ./.github/actions/setup-gradle
167+
with:
168+
java-version: ${{ matrix.java }}
169+
gradle-cache-read-only: ${{ !inputs.is-trunk }}
170+
gradle-cache-write-only: ${{ inputs.is-trunk }}
171+
develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
172+
173+
# If the load-catalog job failed, we won't be able to download the artifact. Since we don't want this to fail
174+
# the overall workflow, so we'll continue here without a test catalog.
175+
- name: Load Test Catalog
176+
id: load-test-catalog
177+
uses: actions/download-artifact@v4
178+
continue-on-error: true
179+
with:
180+
name: combined-test-catalog
181+
182+
- name: JUnit Quarantined Tests
183+
id: junit-quarantined-test
184+
uses: ./.github/actions/run-gradle
185+
with:
186+
test-task: quarantinedTest
187+
timeout-minutes: 180
188+
test-catalog-path: ${{ steps.load-test-catalog.outputs.download-path }}/combined-test-catalog.txt
189+
build-scan-artifact-name: build-scan-quarantined-test-${{ matrix.java }}
190+
191+
- name: JUnit Tests
192+
id: junit-test
193+
uses: ./.github/actions/run-gradle
194+
with:
195+
test-task: test
196+
timeout-minutes: 180 # 3 hours
197+
test-catalog-path: ${{ steps.load-test-catalog.outputs.download-path }}/combined-test-catalog.txt
198+
build-scan-artifact-name: build-scan-test-${{ matrix.java }}
199+
200+
- name: Archive JUnit HTML reports
201+
uses: actions/upload-artifact@v4
202+
id: junit-upload-artifact
203+
with:
204+
name: junit-reports-${{ matrix.java }}
205+
path: |
206+
**/build/reports/tests/*
207+
compression-level: 9
208+
if-no-files-found: ignore
209+
210+
- name: Archive JUnit XML
211+
uses: actions/upload-artifact@v4
212+
with:
213+
name: junit-xml-${{ matrix.java }}
214+
path: |
215+
build/junit-xml/**/*.xml
216+
compression-level: 9
217+
if-no-files-found: ignore
218+
219+
- name: Archive Thread Dumps
220+
id: thread-dump-upload-artifact
221+
if: always() && (steps.junit-test.outputs.gradle-exitcode == '124' || steps.junit-quarantined-test.outputs.gradle-exitcode == '124')
222+
uses: actions/upload-artifact@v4
223+
with:
224+
name: junit-thread-dumps-${{ matrix.java }}
225+
path: |
226+
thread-dumps/*
227+
compression-level: 9
228+
if-no-files-found: ignore
229+
230+
- name: Parse JUnit tests
231+
run: python .github/scripts/junit.py --export-test-catalog ./test-catalog >> $GITHUB_STEP_SUMMARY
232+
env:
233+
GITHUB_WORKSPACE: ${{ github.workspace }}
234+
JUNIT_REPORT_URL: ${{ steps.junit-upload-artifact.outputs.artifact-url }}
235+
THREAD_DUMP_URL: ${{ steps.thread-dump-upload-artifact.outputs.artifact-url }}
236+
GRADLE_TEST_EXIT_CODE: ${{ steps.junit-test.outputs.gradle-exitcode }}
237+
GRADLE_QUARANTINED_TEST_EXIT_CODE: ${{ steps.junit-quarantined-test.outputs.gradle-exitcode }}
238+
239+
- name: Archive Test Catalog
240+
if: ${{ always() && matrix.java == '23' }}
241+
uses: actions/upload-artifact@v4
242+
with:
243+
name: test-catalog
244+
path: test-catalog
245+
compression-level: 9
246+
if-no-files-found: ignore
247+
248+
update-test-catalog:
249+
name: Update Test Catalog
250+
needs: test
251+
if: ${{ always() && inputs.is-trunk && needs.test.outputs.timed-out == 'false' }}
252+
runs-on: ubuntu-latest
253+
permissions:
254+
contents: write
255+
steps:
256+
- name: Checkout Test Catalog
257+
uses: actions/checkout@v4
258+
with:
259+
persist-credentials: true # Needed to commit and push later
260+
ref: test-catalog
261+
- name: Reset Catalog
262+
run: |
263+
rm -rf test-catalog
264+
- name: Download Test Catalog
265+
uses: actions/download-artifact@v4
266+
with:
267+
name: test-catalog
268+
path: test-catalog
269+
- name: Push Test Catalog
270+
# Git user.name and user.email come from https://github.com/actions/checkout?tab=readme-ov-file#push-a-commit-using-the-built-in-token
271+
env:
272+
COMMIT_MSG: |
273+
Update test catalog data for GHA workflow run ${{ github.run_id }}
274+
275+
Commit: https://github.com/apache/kafka/commit/${{ github.sha }}
276+
GitHub Run: https://github.com/apache/kafka/actions/runs/${{ github.run_id }}
277+
run: |
278+
pwd
279+
ls -R
280+
git config user.name 'github-actions[bot]'
281+
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
282+
git add test-catalog
283+
git diff --quiet && git diff --staged --quiet || git commit -m "$COMMIT_MSG"
284+
git push

0 commit comments

Comments
 (0)