name: Integration Tests run-name: Integration Tests ${{ inputs.distinct_run_name && inputs.distinct_run_name || '' }} on: merge_group: pull_request: push: tags: - "*" workflow_dispatch: inputs: cl_ref: description: 'The ref to checkout, defaults to the calling branch' required: false type: string evm-ref: description: 'The sha of the chainlink-evm commit to use if wanted' required: false type: string distinct_run_name: description: 'A unique identifier for this run, only use from other repos' required: false type: string # Only run 1 of this workflow at a time per PR concurrency: group: ${{ github.ref }}-${{ github.repository }}-${{ github.event_name }}--e2e-tests-${{ inputs.distinct_run_name }} cancel-in-progress: true env: # for run-test variables and environment ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ inputs.evm-ref || github.sha }} CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink TEST_SUITE: smoke TEST_ARGS: -test.timeout 12m INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com MOD_CACHE_VERSION: 2 COLLECTION_ID: chainlink-e2e-tests jobs: enforce-ctf-version: name: Enforce CTF Version runs-on: ubuntu-latest # We don't directly merge dependabot PRs, so let's not waste the resources if: github.actor != 'dependabot[bot]' steps: - run: echo "${{github.event_name}}" - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/ccip ref: ${{ inputs.cl_ref }} - name: Check Merge Group Condition id: condition-check run: | echo "Checking event condition..." SHOULD_ENFORCE="false" if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then echo "We are in a merge_group event, now check if we are on the develop branch" target_branch=$(cat $GITHUB_EVENT_PATH | jq -r .merge_group.base_ref) if [[ "$target_branch" == "refs/heads/develop" ]]; then echo "We are on the develop branch, we should enforce ctf version" SHOULD_ENFORCE="true" fi fi echo "should we enforce ctf version = $SHOULD_ENFORCE" echo "should-enforce=$SHOULD_ENFORCE" >> $GITHUB_OUTPUT - name: Enforce CTF Version if: steps.condition-check.outputs.should-enforce == 'true' uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/mod-version@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19 with: go-project-path: ./integration-tests module-name: github.com/smartcontractkit/chainlink-testing-framework/lib enforce-semantic-tag: "true" changes: environment: integration name: Check Paths That Require Tests To Run runs-on: ubuntu-latest # We don't directly merge dependabot PRs, so let's not waste the resources if: github.actor != 'dependabot[bot]' steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/ccip ref: ${{ inputs.cl_ref }} - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: changes with: filters: | changes: - '**/*.go' - '**/*go.sum' - '**/*go.mod' - '.github/workflows/integration-tests.yml' - '**/*Dockerfile' - 'core/**/migrations/*.sql' - 'core/**/config/**/*.toml' - 'integration-tests/**/*.toml' - name: Ignore Filter On Workflow Dispatch if: ${{ github.event_name == 'workflow_dispatch' }} id: ignore-filter run: echo "changes=true" >> $GITHUB_OUTPUT - name: Collect Metrics if: always() id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: id: ${{ env.COLLECTION_ID }}-check-paths org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: Check Paths That Require Tests To Run continue-on-error: true outputs: src: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.changes }} build-lint-integration-tests: name: Build and Lint ${{ matrix.project.name }} runs-on: ubuntu22.04-8cores-32GB # We don't directly merge dependabot PRs, so let's not waste the resources if: github.actor != 'dependabot[bot]' strategy: matrix: project: - name: integration-tests id: e2e path: ./integration-tests cache-id: e2e - name: load id: load path: ./integration-tests/load cache-id: load steps: - name: Collect Metrics id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: id: ${{ env.COLLECTION_ID }}-build-lint-${{ matrix.project.id }} org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: Build and Lint ${{ matrix.project.name }} continue-on-error: true - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/ccip ref: ${{ inputs.cl_ref }} - name: Setup Go uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 with: test_download_vendor_packages_command: cd ${{ matrix.project.path }} && go mod download go_mod_path: ${{ matrix.project.path }}/go.mod cache_key_id: core-${{ matrix.project.cache-id }}-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" - name: Build Go run: | cd ${{ matrix.project.path }} go build ./... go test -run=^# ./... - name: Lint Go uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0 with: version: v1.59.1 # We already cache these directories in setup-go skip-pkg-cache: true skip-build-cache: true # only-new-issues is only applicable to PRs, otherwise it is always set to false only-new-issues: false # disabled for PRs due to unreliability args: --out-format colored-line-number,checkstyle:golangci-lint-report.xml working-directory: ${{ matrix.project.path }} build-chainlink: environment: integration permissions: id-token: write contents: read strategy: matrix: image: - name: "" dockerfile: core/chainlink.Dockerfile tag-suffix: "" - name: (plugins) dockerfile: plugins/chainlink.Dockerfile tag-suffix: -plugins name: Build Chainlink Image ${{ matrix.image.name }} runs-on: ubuntu22.04-8cores-32GB needs: [changes, enforce-ctf-version] steps: - name: Collect Metrics if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: id: ${{ env.COLLECTION_ID }}-build-chainlink org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: Build Chainlink Image ${{ matrix.image.name }} continue-on-error: true - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/ccip ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - name: Setup Github Token if: ${{ inputs.evm-ref }} id: get-gh-token uses: smartcontractkit/.github/actions/setup-github-token@ef78fa97bf3c77de6563db1175422703e9e6674f # setup-github-token@0.2.1 with: aws-role-arn: ${{ secrets.AWS_OIDC_GLOBAL_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }} aws-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }} aws-region: ${{ secrets.AWS_REGION }} set-git-config: "true" - name: Build Chainlink Image if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' uses: ./.github/actions/build-chainlink-image with: tag_suffix: ${{ matrix.image.tag-suffix }} dockerfile: ${{ matrix.image.dockerfile }} git_commit_sha: ${{ inputs.evm-ref || github.sha }} AWS_REGION: ${{ secrets.QA_AWS_REGION }} AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} dep_evm_sha: ${{ inputs.evm-ref }} build-test-image: if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || contains(join(github.event.pull_request.labels.*.name, ' '), 'build-test-image') environment: integration permissions: id-token: write contents: read name: Build Test Image runs-on: ubuntu22.04-16cores-64GB needs: [changes] steps: - name: Collect Metrics if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: id: ${{ env.COLLECTION_ID }}-build-test-image org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: Build Test Image continue-on-error: true - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/ccip ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - name: Build Go Test Command id: build-go-test-command run: | # if the matrix.product.run is set, use it for a different command if [ "${{ matrix.product.run }}" != "" ]; then echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" else echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" fi - name: Setup GAP for Grafana uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 id: setup-gap with: # aws inputs aws-region: ${{ secrets.AWS_REGION }} aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} # other inputs duplicate-authorization-header: "true" eth-smoke-tests-matrix: if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} environment: integration permissions: actions: read checks: write pull-requests: write id-token: write contents: read needs: [build-chainlink, changes, build-lint-integration-tests] env: SELECTED_NETWORKS: SIMULATED CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} CHAINLINK_ENV_USER: ${{ github.actor }} TEST_LOG_LEVEL: debug strategy: fail-fast: false matrix: product: # LM Smoke Test is disabled since project is paused # - name: ccip-lm-smoke # nodes: 1 # os: ubuntu-latest # file: lm # dir: ccip-tests/smoke # run: -run ^TestLmBasic$ - name: ccip-smoke nodes: 1 os: ubuntu-latest file: ccip dir: ccip-tests/smoke run: -run ^TestSmokeCCIPForBidirectionalLane$ - name: ccip-smoke-1.4-pools nodes: 1 os: ubuntu-latest file: ccip dir: ccip-tests/smoke run: -run ^TestSmokeCCIPForBidirectionalLane$ config_path: ./integration-tests/ccip-tests/testconfig/tomls/contract-version1.4.toml - name: ccip-smoke-usdc nodes: 1 os: ubuntu-latest file: ccip dir: ccip-tests/smoke run: -run ^TestSmokeCCIPForBidirectionalLane$ config_path: ./integration-tests/ccip-tests/testconfig/tomls/usdc_mock_deployment.toml - name: ccip-smoke-db-compatibility nodes: 1 os: ubuntu-latest file: ccip dir: ccip-tests/smoke run: -run ^TestSmokeCCIPForBidirectionalLane$ config_path: ./integration-tests/ccip-tests/testconfig/tomls/db-compatibility.toml - name: ccip-smoke-rate-limit nodes: 1 dir: ccip-tests/smoke os: ubuntu-latest file: ccip run: -run ^TestSmokeCCIPRateLimit$ - name: ccip-smoke-rate-limit nodes: 1 dir: ccip-tests/smoke os: ubuntu-latest file: ccip run: -run ^TestSmokeCCIPTokenPoolRateLimits$ - name: ccip-smoke-multicall nodes: 1 dir: ccip-tests/smoke os: ubuntu-latest file: ccip run: -run ^TestSmokeCCIPMulticall$ - name: ccip-smoke-manual-exec nodes: 1 dir: ccip-tests/smoke os: ubuntu-latest file: ccip run: -run ^TestSmokeCCIPManuallyExecuteAfterExecutionFailingDueToInsufficientGas$ - name: ccip-smoke-on-ramp-limits nodes: 1 dir: ccip-tests/smoke os: ubuntu-latest file: ccip run: -run ^TestSmokeCCIPOnRampLimits$ - name: ccip-smoke-off-ramp-capacity nodes: 1 dir: ccip-tests/smoke os: ubuntu-latest file: ccip run: -run ^TestSmokeCCIPOffRampCapacityLimit$ - name: ccip-smoke-off-ramp-agg-rate-limit nodes: 1 dir: ccip-tests/smoke os: ubuntu-latest file: ccip run: -run ^TestSmokeCCIPOffRampAggRateLimit$ - name: ccip-smoke-leader-lane nodes: 15 dir: ccip-tests/smoke os: ubuntu-latest file: ccip run: -run ^TestSmokeCCIPForBidirectionalLane$ config_path: ./integration-tests/ccip-tests/testconfig/tomls/leader-lane.toml - name: ccip-smoke-reorg-bidirectional nodes: 1 dir: ccip-tests/smoke os: ubuntu-latest file: ccip run: -run ^TestSmokeCCIPReorgBelowFinality$ -v config_path: ./integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml - name: ccip-smoke-reorg-below-finality nodes: 1 dir: ccip-tests/smoke os: ubuntu-latest file: ccip run: -run ^TestSmokeCCIPReorgAboveFinalityAtDestination$ -v config_path: ./integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml - name: ccip-smoke-reorg-above-finality nodes: 1 dir: ccip-tests/smoke os: ubuntu-latest file: ccip run: -run ^TestSmokeCCIPReorgAboveFinalityAtSource$ -v config_path: ./integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml - name: runlog id: runlog nodes: 2 os: ubuntu-latest pyroscope_env: "ci-smoke-runlog-evm-simulated" - name: cron id: cron nodes: 2 os: ubuntu-latest pyroscope_env: "ci-smoke-cron-evm-simulated" - name: flux id: flux nodes: 1 os: ubuntu-latest pyroscope_env: "ci-smoke-flux-evm-simulated" - name: ocr id: ocr nodes: 2 os: ubuntu-latest file: ocr pyroscope_env: ci-smoke-ocr-evm-simulated - name: reorg_above_finality id: reorg_above_finality nodes: 1 os: ubuntu-latest file: reorg_above_finality pyroscope_env: ci-smoke-reorg-above-finality-evm-simulated - name: ocr2 id: ocr2 nodes: 6 os: ubuntu22.04-16cores-64GB file: ocr2 pyroscope_env: ci-smoke-ocr2-evm-simulated - name: ocr2 id: ocr2-plugins nodes: 6 os: ubuntu22.04-16cores-64GB pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated tag_suffix: "-plugins" - name: vrf id: vrf nodes: 2 os: ubuntu-latest pyroscope_env: ci-smoke-vrf-evm-simulated - name: vrfv2 id: vrfv2 nodes: 6 os: ubuntu-latest pyroscope_env: ci-smoke-vrf2-evm-simulated - name: vrfv2plus id: vrfv2plus nodes: 9 os: ubuntu-latest pyroscope_env: ci-smoke-vrf2plus-evm-simulated - name: forwarder_ocr id: forwarder_ocr nodes: 2 os: ubuntu-latest pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated - name: forwarders_ocr2 id: forwarders_ocr2 nodes: 2 os: ubuntu-latest pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated runs-on: ${{ matrix.product.os }} name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} steps: # Handy for debugging resource usage # - name: Collect Workflow Telemetry # uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0 - name: Collect Metrics if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: id: ${{ env.COLLECTION_ID }}-matrix-${{ matrix.product.id }} org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' continue-on-error: true - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/ccip ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - name: Build Go Test Command id: build-go-test-command run: | # if dir is provided use it, otherwise use the smoke dir if [ "${{ matrix.product.dir }}" != "" ]; then dir=${{ matrix.product.dir }} else dir=smoke fi # if the matrix.product.run is set, use it for a different command if [ "${{ matrix.product.run }}" != "" ]; then echo "run_command=${{ matrix.product.run }} ./${dir}/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" else echo "run_command=./${dir}/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" fi - name: Check for "enable tracing" label id: check-label run: | label=$(jq -r '.pull_request.labels[]?.name // empty' "$GITHUB_EVENT_PATH") if [[ -n "$label" ]]; then if [[ "$label" == "enable tracing" ]]; then echo "Enable tracing label found." echo "trace=true" >> $GITHUB_OUTPUT else echo "Enable tracing label not found." echo "trace=false" >> $GITHUB_OUTPUT fi else echo "No labels present or labels are null." echo "trace=false" >> $GITHUB_OUTPUT fi - name: Setup Grafana and OpenTelemetry id: docker-setup if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' run: | # Create network docker network create --driver bridge tracing # Make trace directory cd integration-tests/smoke/ mkdir ./traces chmod -R 777 ./traces # Switch directory cd ../../.github/tracing # Create a Docker volume for traces # docker volume create otel-traces # Start OpenTelemetry Collector # Note the user must be set to the same user as the runner for the trace data to be accessible docker run -d --network=tracing --name=otel-collector \ -v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \ -v $PWD/../../integration-tests/smoke/traces:/tracing \ --user "$(id -u):$(id -g)" \ -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml - name: Locate Docker Volume id: locate-volume if: false run: | echo "VOLUME_PATH=$(docker volume inspect --format '{{ .Mountpoint }}' otel-traces)" >> $GITHUB_OUTPUT - name: Show Otel-Collector Logs if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' run: | docker logs otel-collector - name: Set Override Config id: set_override_config run: | # if the matrix.product.config_path is set, use it as the override config if [ "${{ matrix.product.config_path }}" != "" ]; then echo "base_64_override=$(base64 -w 0 -i ${{ matrix.product.config_path }})" >> "$GITHUB_OUTPUT" fi - name: Setup GAP for Grafana uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 id: setup-gap with: # aws inputs aws-region: ${{ secrets.AWS_REGION }} aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} # other inputs duplicate-authorization-header: "true" # metrics inputs metrics-job-name: "grafana" gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 CCIP TOML secrets uses: ./.github/actions/setup-create-base64-config-ccip id: setup_create_base64_config_ccip with: runId: ${{ github.run_id }} pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }} pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} selectedNetworks: SIMULATED_1,SIMULATED_2 chainlinkVersion: ${{ inputs.evm-ref || github.sha }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 env: BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }} TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }} DEBUG_RESTY: false with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} test_config_logging_run_id: ${{ github.run_id }} test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ inputs.evm-ref || github.sha }}${{ matrix.product.tag_suffix }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} artifacts_name: ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}-test-logs artifacts_location: | ./integration-tests/smoke/logs/ ./integration-tests/ccip-tests/smoke/logs/* /tmp/gotest.log publish_check_name: ${{ matrix.product.name }} token: ${{ secrets.GITHUB_TOKEN }} go_mod_path: ./integration-tests/go.mod cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" should_tidy: "false" go_coverage_src_dir: /var/tmp/go-coverage go_coverage_dest_dir: ${{ github.workspace }}/.covdata DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - name: Upload Coverage Data uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 timeout-minutes: 2 continue-on-error: true with: name: cl-node-coverage-data-${{ matrix.product.name }}-${{ matrix.product.tag_suffix }} path: .covdata retention-days: 1 # Run this step when changes that do not need the test to run are made - name: Run Setup if: needs.changes.outputs.src == 'false' uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 with: test_download_vendor_packages_command: cd ./integration-tests && go mod download go_mod_path: ./integration-tests/go.mod cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" should_tidy: "false" - name: Show Otel-Collector Logs if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' run: | docker logs otel-collector - name: Permissions on traces if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' run: | ls -l ./integration-tests/smoke/traces - name: Upload Trace Data if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: trace-data path: ./integration-tests/smoke/traces/trace-data.json - name: Print failed test summary if: always() uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 with: test_directories: ./integration-tests/smoke/,./integration-tests/ccip-tests/smoke/ ### Used to check the required checks box when the matrix completes eth-smoke-tests: if: always() runs-on: ubuntu-latest name: ETH Smoke Tests needs: [eth-smoke-tests-matrix] steps: - name: Check smoke test matrix status if: needs.eth-smoke-tests-matrix.result != 'success' run: | echo "${{ needs.eth-smoke-tests-matrix.result }}" exit 1 - name: Collect Metrics if: always() id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: id: ${{ env.COLLECTION_ID }}-matrix-results org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: ETH Smoke Tests matrix-aggregator-status: ${{ needs.eth-smoke-tests-matrix.result }} continue-on-error: true cleanup: name: Clean up integration environment deployments if: always() needs: [eth-smoke-tests] runs-on: ubuntu-latest steps: - name: Checkout repo if: ${{ github.event_name == 'pull_request' }} uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/ccip ref: ${{ inputs.cl_ref }} - name: 🧼 Clean up Environment if: ${{ github.event_name == 'pull_request' }} uses: ./.github/actions/delete-deployments with: environment: integration ref: ${{ github.head_ref }} # See https://github.com/github/docs/issues/15319#issuecomment-1476705663 - name: Collect Metrics if: ${{ github.event_name == 'pull_request' }} id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: id: ${{ env.COLLECTION_ID }}-env-cleanup org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: Clean up integration environment deployments continue-on-error: true show-coverage: name: Show Chainlink Node Go Coverage if: always() needs: [cleanup] runs-on: ubuntu-latest steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/ccip ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - name: Download All Artifacts uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6 with: path: cl-node-coverage-data pattern: cl-node-coverage-data-* merge-multiple: true - name: Show Coverage run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl-node-coverage-data/*/merged" # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss # this will also only run if both of the matrix jobs pass eth-smoke-go-mod-cache: environment: integration needs: [eth-smoke-tests] runs-on: ubuntu-latest name: ETH Smoke Tests Go Mod Cache continue-on-error: true steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/ccip ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - name: Run Setup uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 with: test_download_vendor_packages_command: | cd ./integration-tests go mod download # force download of test dependencies go test -run=NonExistentTest ./smoke/... || echo "ignore expected test failure" go_mod_path: ./integration-tests/go.mod cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "false" ### Migration tests node-migration-tests: name: Version Migration Tests environment: integration permissions: checks: write pull-requests: write id-token: write contents: read runs-on: ubuntu-latest needs: [build-chainlink, changes] # Only run migration tests on new tags if: startsWith(github.ref, 'refs/tags/') env: SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} CHAINLINK_ENV_USER: ${{ github.actor }} CHAINLINK_IMAGE: public.ecr.aws/w0i8p0z9/chainlink-ccip UPGRADE_VERSION: ${{ inputs.evm-ref || github.sha }} UPGRADE_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink TEST_LOG_LEVEL: debug TEST_SUITE: migration steps: - name: Collect Metrics id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: id: ${{ env.COLLECTION_ID }}-migration-tests org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: Version Migration Tests test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' continue-on-error: true - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/ccip ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - name: Get Latest Version id: get_latest_version run: | untrimmed_ver=$(curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .name) latest_version="${untrimmed_ver:1}" # Check if latest_version is empty if [ -z "$latest_version" ]; then echo "Error: The latest_version is empty. The migration tests need a verison to run." exit 1 fi # CCIP uses the `-release` suffix for their versions echo "latest_version=${latest_version}-release" >> "$GITHUB_OUTPUT" - name: Name Versions run: | echo "Running migration tests from version '${{ steps.get_latest_version.outputs.latest_version }}' to: '${{ inputs.evm-ref || github.sha }}'" - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-upgrade-config with: selectedNetworks: ${{ env.SELECTED_NETWORKS }} chainlinkVersion: ${{ steps.get_latest_version.outputs.latest_version }} upgradeVersion: ${{ env.UPGRADE_VERSION }} runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - name: Run Migration Tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 with: test_command_to_run: cd ./integration-tests && go test -timeout 20m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ steps.get_latest_version.outputs.latest_version }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} artifacts_name: node-migration-test-logs artifacts_location: | ./integration-tests/migration/logs /tmp/gotest.log publish_check_name: Node Migration Test Results token: ${{ secrets.GITHUB_TOKEN }} go_mod_path: ./integration-tests/go.mod cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" go_coverage_src_dir: /var/tmp/go-coverage go_coverage_dest_dir: ${{ github.workspace }}/.covdata should_tidy: "false" DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ env.UPGRADE_IMAGE }} DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - name: Upload Coverage Data uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 timeout-minutes: 2 continue-on-error: true with: name: cl-node-coverage-data-migration-tests path: .covdata retention-days: 1 - name: Notify Slack if: failure() && github.event_name != 'workflow_dispatch' uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 env: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} with: channel-id: "C05QQ8Z342V" slack-message: "CCIP Node Migration Tests Failed :x: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}" payload: | { "attachments": [ { "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}", "blocks": [ { "type": "header", "text": { "type": "plain_text", "text": "CCIP Version Migration Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", "emoji": true } }, { "type": "section", "text": { "type": "mrkdwn", "text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed! Notifying ' || 'All Good!' }}" } }, { "type": "divider" }, { "type": "section", "text": { "type": "mrkdwn", "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>" } } ] } ] }