name: CCIP Chaos Tests on: workflow_run: workflows: [ CCIP Load Test ] types: [ completed ] branches: [ ccip-develop ] workflow_dispatch: # Only run 1 of this workflow at a time per PR concurrency: group: chaos-ccip-tests-chainlink-${{ github.ref }} cancel-in-progress: true env: CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} MOD_CACHE_VERSION: 1 jobs: build-chainlink: environment: integration permissions: id-token: write contents: read name: Build Chainlink Image runs-on: ubuntu20.04-16cores-64GB steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Check if image exists id: check-image uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 with: repository: chainlink tag: ${{ github.sha }} AWS_REGION: ${{ secrets.QA_AWS_REGION }} AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Build Image if: steps.check-image.outputs.exists == 'false' uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 env: GH_TOKEN: ${{ github.token }} with: cl_repo: smartcontractkit/chainlink-ccip cl_ref: ${{ github.sha }} push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ github.sha }} QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Collect Metrics if: always() id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: id: ccip-chaos-tests-build-chainlink-image org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: Build Chainlink Image continue-on-error: true build-test-image: environment: integration permissions: id-token: write contents: read name: Build Test Image runs-on: ubuntu20.04-16cores-64GB steps: - name: Collect Metrics id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: id: ccip-chaos-tests-build-test-image org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: Build Test Image continue-on-error: true - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Build Test Image uses: ./.github/actions/build-test-image with: QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} ccip-chaos-tests: environment: integration permissions: issues: read checks: write pull-requests: write id-token: write contents: read name: CCIP Chaos Tests runs-on: ubuntu-latest needs: [ build-chainlink, build-test-image ] env: TEST_SUITE: chaos TEST_ARGS: -test.timeout 30m CHAINLINK_COMMIT_SHA: ${{ github.sha }} CHAINLINK_ENV_USER: ${{ github.actor }} TEST_TRIGGERED_BY: ccip-cron-chaos-eth TEST_LOG_LEVEL: debug DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable GH_TOKEN: ${{ github.token }} steps: - name: Collect Metrics id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: id: ccip-chaos-tests org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: CCIP Chaos Tests test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' continue-on-error: true - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Prepare Base64 TOML override for CCIP secrets uses: ./.github/actions/setup-create-base64-config-ccip id: setup_create_base64_config_ccip with: runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} chainlinkVersion: ${{ github.sha }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - name: Run Chaos Tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 env: BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} with: test_command_to_run: cd ./integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 -run 'TestChaosCCIP' ./chaos 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci test_download_vendor_packages_command: make gomod artifacts_location: ./integration-tests/chaos/logs publish_check_name: CCIP Chaos Test Results publish_report_paths: ./tests-chaos-report.xml triggered_by: ${{ env.TEST_TRIGGERED_BY }} token: ${{ secrets.GITHUB_TOKEN }} go_mod_path: ./integration-tests/go.mod QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} CGO_ENABLED: "1" aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} DEFAULT_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" ## Notify in slack if the job fails - name: Notify Slack if: failure() && github.event_name != 'workflow_dispatch' uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 env: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} with: channel-id: "#ccip-testing" slack-message: ":x: :mild-panic-intensifies: CCIP chaos tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}" ## Run Cleanup if the job succeeds - name: cleanup if: always() uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 with: triggered_by: ${{ env.TEST_TRIGGERED_BY }} ccip-chaos-with-load-tests: environment: integration permissions: issues: read checks: write pull-requests: write id-token: write contents: read name: CCIP Load With Chaos Tests if: false # Disabled until CCIP-2555 is resolved runs-on: ubuntu-latest needs: [ build-chainlink, build-test-image ] env: TEST_SUITE: load TEST_ARGS: -test.timeout 1h CHAINLINK_COMMIT_SHA: ${{ github.sha }} CHAINLINK_ENV_USER: ${{ github.actor }} TEST_TRIGGERED_BY: ccip-cron-chaos-and-load-eth TEST_LOG_LEVEL: debug DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable GH_TOKEN: ${{ github.token }} steps: - name: Collect Metrics id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: id: ccip-chaos-tests-with-load-test org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: CCIP load with chaos test continue-on-error: true - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Prepare Base64 TOML override for CCIP secrests uses: ./.github/actions/setup-create-base64-config-ccip id: setup_create_base64_config_ccip with: runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} chainlinkVersion: ${{ github.sha }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - name: Run Load With Chaos Tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 env: BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} with: test_command_to_run: cd ./integration-tests/ccip-tests && go test -timeout 2h -count=1 -json -test.parallel 4 -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' ./load 2>&1 | tee /tmp/gotest.log | gotestfmt test_download_vendor_packages_command: make gomod artifacts_location: ./integration-tests/load/logs publish_check_name: CCIP Chaos With Load Test Results publish_report_paths: ./tests-chaos-with-load-report.xml triggered_by: ${{ env.TEST_TRIGGERED_BY }} token: ${{ secrets.GITHUB_TOKEN }} go_mod_path: ./integration-tests/go.mod QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} CGO_ENABLED: "1" aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} DEFAULT_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} DEFAULT_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests" ## Notify in slack if the job fails - name: Notify Slack if: failure() && github.event_name != 'workflow_dispatch' uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 env: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} with: channel-id: "#ccip-testing" slack-message: ":x: :mild-panic-intensifies: CCIP chaos with load tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}" ## Run Cleanup if the job succeeds - name: cleanup if: always() uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 with: triggered_by: ${{ env.TEST_TRIGGERED_BY }}