Skip to content

Update CRDS cache workflow usage to reduce random failures #9040

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
braingram merged 3 commits into from
Jan 2, 2025
Merged

Update CRDS cache workflow usage to reduce random failures #9040

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
151ff41
Revert "hotfix for merge conflict"
braingram Jan 2, 2025
031358a
Revert "[CI] set default CRDS context to `jwst-edit` (#8987)"
braingram Jan 2, 2025
e82bac9
use crds client for getting edit context
braingram Jan 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 16 additions & 5 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ on:
type: string
required: false
default: ''
crds_server_url:
description: CRDS server URL
crds_server:
description: CRDS server
type: string
required: false
default: https://jwst-crds.stsci.edu
Expand All @@ -46,17 +46,28 @@ jobs:
envs: |
- linux: check-dependencies
- linux: check-types
latest_crds_contexts:
uses: ./.github/workflows/contexts.yml
crds_context:
needs: [ latest_crds_contexts ]
runs-on: ubuntu-latest
steps:
- id: context
run: echo context=${{ github.event_name == 'workflow_dispatch' && (inputs.crds_context != '' && inputs.crds_context || needs.latest_crds_contexts.outputs.jwst) || needs.latest_crds_contexts.outputs.jwst }} >> $GITHUB_OUTPUT
outputs:
context: ${{ steps.context.outputs.context }}
test:
uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@9f1f43251dde69da8613ea8e11144f05cdea41d5 # v1.15.0
needs: [ crds_context ]
with:
setenv: |
CRDS_PATH: /tmp/data/crds_cache
CRDS_SERVER_URL: ${{ inputs.crds_server_url || 'https://jwst-crds.stsci.edu' }}
CRDS_CONTEXT: ${{ inputs.crds_context || 'jwst-edit' }}
CRDS_SERVER_URL: ${{ github.event_name == 'workflow_dispatch' && inputs.crds_server || 'https://jwst-crds.stsci.edu' }}
CRDS_CONTEXT: ${{ needs.crds_context.outputs.context }}
CRDS_CLIENT_RETRY_COUNT: 3
CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
cache-path: /tmp/data/crds_cache
cache-key: crds-${{ inputs.crds_context || 'jwst-edit' }}
cache-key: crds-${{ needs.crds_context.outputs.context }}
envs: |
- linux: py310-oldestdeps-xdist-cov
pytest-results-summary: true
Expand Down
19 changes: 15 additions & 4 deletions .github/workflows/ci_cron.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ on:
type: string
required: false
default: ''
crds_server_url:
description: CRDS server URL
crds_server:
description: CRDS server
type: string
required: false
default: https://jwst-crds.stsci.edu
Expand All @@ -22,14 +22,25 @@ concurrency:
cancel-in-progress: true

jobs:
latest_crds_contexts:
uses: ./.github/workflows/contexts.yml
crds_context:
needs: [ latest_crds_contexts ]
runs-on: ubuntu-latest
steps:
- id: context
run: echo context=${{ github.event_name == 'workflow_dispatch' && (inputs.crds_context != '' && inputs.crds_context || needs.latest_crds_contexts.outputs.jwst) || needs.latest_crds_contexts.outputs.jwst }} >> $GITHUB_OUTPUT
outputs:
context: ${{ steps.context.outputs.context }}
test:
if: (github.repository == 'spacetelescope/jwst' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run scheduled tests')))
uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@9f1f43251dde69da8613ea8e11144f05cdea41d5 # v1.15.0
needs: [ crds_context ]
with:
setenv: |
CRDS_PATH: /tmp/crds_cache
CRDS_SERVER_URL: ${{ inputs.crds_server_url || 'https://jwst-crds.stsci.edu' }}
CRDS_CONTEXT: ${{ inputs.crds_context || 'jwst-edit' }}
CRDS_SERVER_URL: ${{ github.event_name == 'workflow_dispatch' && inputs.crds_server || 'https://jwst-crds.stsci.edu' }}
CRDS_CONTEXT: ${{ needs.crds_context.outputs.context }}
CRDS_CLIENT_RETRY_COUNT: 3
CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
cache-path: /tmp/crds_cache
Expand Down
24 changes: 24 additions & 0 deletions .github/workflows/contexts.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
name: contexts

on:
workflow_call:
outputs:
jwst:
value: ${{ jobs.contexts.outputs.jwst }}
workflow_dispatch:

jobs:
contexts:
name: retrieve latest CRDS contexts
runs-on: ubuntu-latest
outputs:
jwst: ${{ steps.jwst_crds_context.outputs.pmap }}
steps:
- id: jwst_crds_context
env:
OBSERVATORY: jwst
CRDS_SERVER_URL: https://jwst-crds.stsci.edu
run: |
pip install crds
echo "pmap=$(crds list --resolve-contexts --contexts jwst-edit)" >> $GITHUB_OUTPUT
- run: if [[ ! -z "${{ steps.jwst_crds_context.outputs.pmap }}" ]]; then echo ${{ steps.jwst_crds_context.outputs.pmap }}; else exit 1; fi
19 changes: 15 additions & 4 deletions .github/workflows/tests_devdeps.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ on:
type: string
required: false
default: ''
crds_server_url:
description: CRDS server URL
crds_server:
description: CRDS server
type: string
required: false
default: https://jwst-crds.stsci.edu
Expand All @@ -31,14 +31,25 @@ concurrency:
cancel-in-progress: true

jobs:
latest_crds_contexts:
uses: ./.github/workflows/contexts.yml
crds_context:
needs: [ latest_crds_contexts ]
runs-on: ubuntu-latest
steps:
- id: context
run: echo context=${{ github.event_name == 'workflow_dispatch' && (inputs.crds_context != '' && inputs.crds_context || needs.latest_crds_contexts.outputs.jwst) || needs.latest_crds_contexts.outputs.jwst }} >> $GITHUB_OUTPUT
outputs:
context: ${{ steps.context.outputs.context }}
test:
if: (github.repository == 'spacetelescope/jwst' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests')))
uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@9f1f43251dde69da8613ea8e11144f05cdea41d5 # v1.15.0
needs: [ crds_context ]
with:
setenv: |
CRDS_PATH: /tmp/data/crds_cache
CRDS_SERVER_URL: ${{ inputs.crds_server_url || 'https://jwst-crds.stsci.edu' }}
CRDS_CONTEXT: ${{ inputs.crds_context || 'jwst-edit' }}
CRDS_SERVER_URL: ${{ github.event_name == 'workflow_dispatch' && inputs.crds_server || 'https://jwst-crds.stsci.edu' }}
CRDS_CONTEXT: ${{ needs.crds_context.outputs.context }}
CRDS_CLIENT_RETRY_COUNT: 3
CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
cache-path: /tmp/data/crds_cache
Expand Down
Loading