The OSS Review Toolkit (ORT) #11080
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: The OSS Review Toolkit (ORT) | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| actions: read | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| branch_name: | |
| description: "The branch to run against the ORT tool" | |
| required: true | |
| default: "main" | |
| jobs: | |
| run-ort: | |
| name: Create attribution files | |
| runs-on: ubuntu-latest | |
| # 1. For workflow_dispatch, always allow | |
| # 2. For pull_request, run if branch is not autogenerated ort-diff-for- branches | |
| if: > | |
| github.repository_owner == 'valkey-io' && | |
| (github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && !startsWith(github.head_ref, 'ort-diff-for-'))) | |
| continue-on-error: true | |
| env: | |
| PYTHON_ASYNC_ATTRIBUTIONS: "python/glide-async/THIRD_PARTY_LICENSES_PYTHON_ASYNC" | |
| PYTHON_SYNC_ATTRIBUTIONS: "python/glide-sync/THIRD_PARTY_LICENSES_PYTHON_SYNC" | |
| NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE" | |
| RUST_ATTRIBUTIONS: "glide-core/THIRD_PARTY_LICENSES_RUST" | |
| JAVA_ATTRIBUTIONS: "java/THIRD_PARTY_LICENSES_JAVA" | |
| GO_ATTRIBUTIONS: "go/THIRD_PARTY_LICENSES_GO" | |
| INPUT_TARGET_BRANCH: ${{ github.event.inputs.branch_name }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| HEAD_REF: ${{ github.head_ref }} | |
| steps: | |
| - name: Setup target branch and commit | |
| run: | | |
| if [ "$EVENT_NAME" == 'workflow_dispatch' ]; then | |
| echo "TARGET_BRANCH=$INPUT_TARGET_BRANCH" >> $GITHUB_ENV | |
| elif [ "$EVENT_NAME" == 'pull_request' ]; then | |
| echo "TARGET_BRANCH=$HEAD_REF" >> $GITHUB_ENV | |
| fi | |
| - name: Checkout target branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.TARGET_BRANCH }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| fetch-depth: 0 # Fetch all history for all branches and tags | |
| - name: Setup target commit | |
| run: | | |
| echo "TARGET_COMMIT=`git rev-parse HEAD`" >> $GITHUB_ENV | |
| - name: Set up JDK 11 for the ORT package | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: "temurin" | |
| java-version: 11 | |
| - name: Cache ORT and Gradle packages | |
| uses: actions/cache@v4 | |
| id: cache-ort | |
| with: | |
| path: | | |
| ./ort | |
| ~/.gradle/caches | |
| ~/.gradle/wrapper | |
| key: ${{ runner.os }}-ort | |
| - name: Checkout ORT Repository | |
| if: steps.cache-ort.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: "oss-review-toolkit/ort" | |
| path: "./ort" | |
| ref: "46.0.0" | |
| submodules: recursive | |
| - name: Install Rust toolchain | |
| uses: dtolnay/[email protected] | |
| - name: Build and install ORT | |
| if: steps.cache-ort.outputs.cache-hit != 'true' | |
| working-directory: ./ort/ | |
| run: | | |
| export JAVA_OPTS="$JAVA_OPTS -Xmx8g" | |
| ./gradlew installDist | |
| - name: Create ORT config file | |
| run: | | |
| mkdir -p ~/.ort/config | |
| cat << EOF > ~/.ort/config/config.yml | |
| ort: | |
| analyzer: | |
| skip_excluded: true | |
| allowDynamicVersions: true | |
| enabledPackageManagers: [Cargo, GoMod, NPM, PIP, GradleInspector] | |
| EOF | |
| cat ~/.ort/config/config.yml | |
| ### NodeJS ### | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v5 | |
| - name: Run ORT tools for Node | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/node" | |
| ### Python ### | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install python-inspector | |
| working-directory: ./python | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install git+https://github.com/nexB/python-inspector | |
| - name: Install dev requirements | |
| working-directory: ./python | |
| run: | | |
| python -m pip install -r dev_requirements.txt | |
| - name: Run ORT tools for Python Async Client | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/python/glide-async" | |
| - name: Run ORT tools for Python Sync Client | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/python/glide-sync" | |
| ### Rust glide-core ### | |
| - name: Run ORT tools for glide-core | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/glide-core" | |
| ### Rust FFI ### | |
| - name: Run ORT tools for FFI | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/ffi" | |
| ### Java ### | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: "temurin" | |
| java-version: 11 | |
| - name: Install protoc (protobuf) | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| version: "29.1" | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build java artifact | |
| working-directory: ./java | |
| run: | | |
| ./gradlew publishToMavenLocal -x buildRust -x javadoc | |
| - name: Run ORT tools for Java | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/java" | |
| ### GO ### | |
| - name: Set up GO | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: 1.22 | |
| - name: Generate protobuf for GO | |
| working-directory: ./go | |
| run: | | |
| go install google.golang.org/protobuf/cmd/[email protected] | |
| make generate-protobuf | |
| - name: Run ORT tools for Go | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/go" | |
| ### Get licenses ### | |
| - name: Retrieve licenses list | |
| working-directory: ./utils | |
| run: | | |
| { | |
| echo 'LICENSES_LIST<<EOF' | |
| python3 get_licenses_from_ort.py | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| ### Upload licenses ### | |
| - name: Get current date | |
| id: date | |
| run: | | |
| CURR_DATE=$(date +'%Y-%m-%d-%H') | |
| echo "date=${CURR_DATE}" >> $GITHUB_OUTPUT | |
| - name: Upload the final package list | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: final-package-list-${{ steps.date.outputs.date }} | |
| path: | | |
| utils/final_package_list.txt | |
| retention-days: 30 | |
| - name: Upload the skipped package list | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: skipped-package-list-${{ steps.date.outputs.date }} | |
| path: | | |
| utils/skipped_package_list.txt | |
| retention-days: 30 | |
| - name: Upload the unknown/unapproved package list | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: unapproved-package-list-${{ steps.date.outputs.date }} | |
| path: | | |
| utils/unapproved_package_list.txt | |
| retention-days: 30 | |
| - name: Check for unapproved packages | |
| run: | | |
| if [ -s utils/unapproved_package_list.txt ]; then | |
| echo "::error::Found unapproved packages. Please review utils/unapproved_package_list.txt" | |
| cat utils/unapproved_package_list.txt | |
| exit 1 | |
| else | |
| echo "No unapproved packages found." | |
| fi | |
| ### Check for attributions diff ### (FFI attributions merged to FFI clients' attributions) | |
| - name: Check for diff | |
| run: | | |
| cp python/glide-async/ort_results/NOTICE_DEFAULT $PYTHON_ASYNC_ATTRIBUTIONS | |
| cp python/glide-sync/ort_results/NOTICE_DEFAULT $PYTHON_SYNC_ATTRIBUTIONS | |
| cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS | |
| cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS | |
| cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS | |
| : Workaround for empty attributions for go packages. See discussions in https://github.com/valkey-io/valkey-glide/pull/3469 and in https://github.com/oss-review-toolkit/ort/issues/10454 | |
| curl https://raw.githubusercontent.com/valkey-io/valkey-glide/9c9ef0ae2c2ee865c7867f6ac0d3f9aac054bac8/go/THIRD_PARTY_LICENSES_GO -o $GO_ATTRIBUTIONS | |
| cat go/ort_results/NOTICE_DEFAULT >> $GO_ATTRIBUTIONS | |
| : Merge FFI attributions into GO and PYTHON_SYNC attributions | |
| cat ffi/ort_results/NOTICE_DEFAULT >> $GO_ATTRIBUTIONS | |
| cat ffi/ort_results/NOTICE_DEFAULT >> $PYTHON_SYNC_ATTRIBUTIONS | |
| GIT_DIFF=`git diff $PYTHON_ASYNC_ATTRIBUTIONS $PYTHON_SYNC_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS $GO_ATTRIBUTIONS` | |
| if [ -n "$GIT_DIFF" ]; then | |
| echo "FOUND_DIFF=true" >> $GITHUB_ENV | |
| else | |
| echo "FOUND_DIFF=false" >> $GITHUB_ENV | |
| fi | |
| ### Create PR, Note a potential race on the source branch ### | |
| - name: Create pull request | |
| if: ${{ env.FOUND_DIFF == 'true' && github.event_name != 'pull_request' }} | |
| run: | | |
| export ORT_DIFF_BRANCH_NAME="ort-diff-for-$TARGET_BRANCH" | |
| echo "Creating pull request from branch $ORT_DIFF_BRANCH_NAME to branch $TARGET_BRANCH" | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "ort-bot" | |
| git checkout -b ${ORT_DIFF_BRANCH_NAME} | |
| git add $PYTHON_ASYNC_ATTRIBUTIONS $PYTHON_SYNC_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS $GO_ATTRIBUTIONS | |
| git commit -m "Updated attribution files" -s | |
| git push --set-upstream origin ${ORT_DIFF_BRANCH_NAME} -f || { echo "Failed to push branch."; exit 1; } | |
| # Check if PR already exists | |
| existing_pr=$(gh pr list --base ${TARGET_BRANCH} --head ${ORT_DIFF_BRANCH_NAME} --json number --jq '.[0].number') | |
| if [ -z "$existing_pr" ]; then | |
| # Create a new PR if none exists | |
| title="Updated attribution files for commit ${TARGET_COMMIT}" | |
| gh pr create -B ${TARGET_BRANCH} -H ${ORT_DIFF_BRANCH_NAME} --title "${title}" --body "Created by Github action. ${{ env.LICENSES_LIST }}" || { echo "Failed to create PR."; exit 1; } | |
| echo "Pull request created successfully." | |
| else | |
| # Update the existing PR | |
| echo "Pull request #$existing_pr already exists. Updating branch." | |
| gh pr edit $existing_pr --title "Updated attribution files for commit ${TARGET_COMMIT}" --body "Created by Github action. ${{ env.LICENSES_LIST }}" || { echo "Failed to update PR."; exit 1; } | |
| echo "Pull request updated successfully." | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| INPUT_VERSION: ${{ github.event.inputs.version }} | |
| ### Warn of outdated attributions for PR ### | |
| - name: Warn of outdated attributions due to the PR | |
| if: ${{ env.FOUND_DIFF == 'true' && github.event_name == 'pull_request' }} | |
| run: | | |
| ATTRIBUTION_FILES=( | |
| "${{ env.PYTHON_ASYNC_ATTRIBUTIONS }}" | |
| "${{ env.PYTHON_SYNC_ATTRIBUTIONS }}" | |
| "${{ env.NODE_ATTRIBUTIONS }}" | |
| "${{ env.RUST_ATTRIBUTIONS }}" | |
| "${{ env.JAVA_ATTRIBUTIONS }}" | |
| "${{ env.GO_ATTRIBUTIONS }}" | |
| ) | |
| MESSAGE="WARNING! The attribution files differ in this PR. Please ensure an updating PR is issued using a scheduled or manual run of this workflow!" | |
| # Echo the message to the console | |
| echo "$MESSAGE" | |
| # Emit a general warning in the action log | |
| echo "::warning::$MESSAGE" | |
| # Loop through the attribution files | |
| for FILE in "${ATTRIBUTION_FILES[@]}"; do | |
| if git diff --quiet "$FILE"; then | |
| continue | |
| else | |
| # Emit a warning associated with the changed file | |
| echo "::warning file=$FILE::WARNING! The attribution file '$FILE' differs in this PR." | |
| fi | |
| done |