tailscale/.github/workflows/test.yml

# This is our main "CI tests" workflow. It runs everything that should run on
# both PRs and merged commits, and for the latter reports failures to slack.
name: CI

env:
  # Our fuzz job, powered by OSS-Fuzz, fails periodically because we upgrade to
  # new Go versions very eagerly. OSS-Fuzz is a little more conservative, and
  # ends up being unable to compile our code.
  #
  # When this happens, we want to disable the fuzz target until OSS-Fuzz catches
  # up. However, we also don't want to forget to turn it back on when OSS-Fuzz
  # can once again build our code.
  #
  # This variable toggles the fuzz job between two modes:
  #  - false: we expect fuzzing to be happy, and should report failure if it's not.
  #  - true: we expect fuzzing is broken, and should report failure if it start working.
  TS_FUZZ_CURRENTLY_BROKEN: false

on:
  push:
    branches:
      - "main"
      - "release-branch/*"
  pull_request:
    branches:
      - "*"
  merge_group:
    branches:
      - "main"

concurrency:
  # For PRs, later CI runs preempt previous ones. e.g. a force push on a PR
  # cancels running CI jobs and starts all new ones.
  #
  # For non-PR pushes, concurrency.group needs to be unique for every distinct
  # CI run we want to have happen. Use run_id, which in practice means all
  # non-PR CI runs will be allowed to run without preempting each other.
  group: ${{ github.workflow }}-$${{ github.pull_request.number || github.run_id }}
  cancel-in-progress: true

jobs:
  race-root-integration:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false # don't abort the entire matrix if one element fails
      matrix:
        include:
          - shard: '1/4'
          - shard: '2/4'
          - shard: '3/4'
          - shard: '4/4'
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: build test wrapper
      run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
    - name: integration tests as root
      run: PATH=$PWD/tool:$PATH /tmp/testwrapper -exec "sudo -E" -race ./tstest/integration/
      env:
        TS_TEST_SHARD: ${{ matrix.shard }}

  test:
    strategy:
      fail-fast: false # don't abort the entire matrix if one element fails
      matrix:
        include:
          - goarch: amd64
          - goarch: amd64
            buildflags: "-race"
            shard: '1/3'
          - goarch: amd64
            buildflags: "-race"
            shard: '2/3'
          - goarch: amd64
            buildflags: "-race"
            shard: '3/3'
          - goarch: "386" # thanks yaml
    runs-on: ubuntu-22.04
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: Restore Cache
      uses: actions/cache@v3
      with:
        # Note: unlike the other setups, this is only grabbing the mod download
        # cache, rather than the whole mod directory, as the download cache
        # contains zips that can be unpacked in parallel faster than they can be
        # fetched and extracted by tar
        path: |
          ~/.cache/go-build
          ~/go/pkg/mod/cache
          ~\AppData\Local\go-build          
        # The -2- here should be incremented when the scheme of data to be
        # cached changes (e.g. path above changes).
        key: ${{ github.job }}-${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }}
        restore-keys: |
          ${{ github.job }}-${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-2-${{ hashFiles('**/go.sum') }}
          ${{ github.job }}-${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-2-          
    - name: build all
      if: matrix.buildflags == '' # skip on race builder
      run: ./tool/go build ${{matrix.buildflags}} ./...
      env:
        GOARCH: ${{ matrix.goarch }}
    - name: build variant CLIs
      if: matrix.buildflags == '' # skip on race builder
      run: |
        export TS_USE_TOOLCHAIN=1
        ./build_dist.sh --extra-small ./cmd/tailscaled
        ./build_dist.sh --box ./cmd/tailscaled
        ./build_dist.sh --extra-small --box ./cmd/tailscaled
        rm -f tailscaled        
      env:
        GOARCH: ${{ matrix.goarch }}
    - name: get qemu # for tstest/archtest
      if: matrix.goarch == 'amd64' && matrix.buildflags == ''
      run: |
        sudo apt-get -y update
        sudo apt-get -y install qemu-user        
    - name: build test wrapper
      run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
    - name: test all
      run: PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}}
      env:
        GOARCH: ${{ matrix.goarch }}
        TS_TEST_SHARD: ${{ matrix.shard }}
    - name: bench all
      run: ./tool/go test ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$ $(for x in $(git grep -l "^func Benchmark" | xargs dirname | sort | uniq); do echo "./$x"; done)
      env:
        GOARCH: ${{ matrix.goarch }}
    - name: check that no tracked files changed
      run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1)
    - name: check that no new files were added
      run: |
        # Note: The "error: pathspec..." you see below is normal!
        # In the success case in which there are no new untracked files,
        # git ls-files complains about the pathspec not matching anything.
        # That's OK. It's not worth the effort to suppress. Please ignore it.
        if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*'
        then
          echo "Build/test created untracked files in the repo (file names above)."
          exit 1
        fi        

  windows:
    runs-on: windows-2022
    steps:
    - name: checkout
      uses: actions/checkout@v4

    - name: Install Go
      uses: actions/setup-go@v4
      with:
        go-version-file: go.mod
        cache: false

    - name: Restore Cache
      uses: actions/cache@v3
      with:
        # Note: unlike the other setups, this is only grabbing the mod download
        # cache, rather than the whole mod directory, as the download cache
        # contains zips that can be unpacked in parallel faster than they can be
        # fetched and extracted by tar
        path: |
          ~/.cache/go-build
          ~/go/pkg/mod/cache
          ~\AppData\Local\go-build          
        # The -2- here should be incremented when the scheme of data to be
        # cached changes (e.g. path above changes).
        key: ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }}
        restore-keys: |
          ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }}
          ${{ github.job }}-${{ runner.os }}-go-2-          
    - name: test
      run: go run ./cmd/testwrapper ./...
    - name: bench all
      # Don't use -bench=. -benchtime=1x.
      # Somewhere in the layers (powershell?)
      # the equals signs cause great confusion.
      run: go test ./... -bench . -benchtime 1x -run "^$"

  vm:
    runs-on: ["self-hosted", "linux", "vm"]
    # VM tests run with some privileges, don't let them run on 3p PRs.
    if: github.repository == 'tailscale/tailscale'
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: Run VM tests
      run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2004
      env:
        HOME: "/tmp"
        TMPDIR: "/tmp"
        XDB_CACHE_HOME: "/var/lib/ghrunner/cache"

  race-build:
    runs-on: ubuntu-22.04
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: build all
      run: ./tool/go install -race ./cmd/...
    - name: build tests
      run: ./tool/go test -race -exec=true ./...

  cross: # cross-compile checks, build only.
    strategy:
      fail-fast: false # don't abort the entire matrix if one element fails
      matrix:
        include:
          # Note: linux/amd64 is not in this matrix, because that goos/goarch is
          # tested more exhaustively in the 'test' job above.
          - goos: linux
            goarch: arm64
          - goos: linux
            goarch: "386" # thanks yaml
          - goos: linux
            goarch: loong64
          - goos: linux
            goarch: arm
            goarm: "5"
          - goos: linux
            goarch: arm
            goarm: "7"
          # macOS
          - goos: darwin
            goarch: amd64
          - goos: darwin
            goarch: arm64
          # Windows
          - goos: windows
            goarch: amd64
          - goos: windows
            goarch: arm64
          # BSDs
          - goos: freebsd
            goarch: amd64
          - goos: openbsd
            goarch: amd64
          # Plan9
          - goos: plan9
            goarch: amd64

    runs-on: ubuntu-22.04
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: Restore Cache
      uses: actions/cache@v3
      with:
        # Note: unlike the other setups, this is only grabbing the mod download
        # cache, rather than the whole mod directory, as the download cache
        # contains zips that can be unpacked in parallel faster than they can be
        # fetched and extracted by tar
        path: |
          ~/.cache/go-build
          ~/go/pkg/mod/cache
          ~\AppData\Local\go-build          
        # The -2- here should be incremented when the scheme of data to be
        # cached changes (e.g. path above changes).
        key: ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }}
        restore-keys: |
          ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-${{ hashFiles('**/go.sum') }}
          ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-          
    - name: build all
      run: ./tool/go build ./cmd/...
      env:
        GOOS: ${{ matrix.goos }}
        GOARCH: ${{ matrix.goarch }}
        GOARM: ${{ matrix.goarm }}
        CGO_ENABLED: "0"
    - name: build tests
      run: ./tool/go test -exec=true ./...
      env:
        GOOS: ${{ matrix.goos }}
        GOARCH: ${{ matrix.goarch }}
        CGO_ENABLED: "0"

  ios: # similar to cross above, but iOS can't build most of the repo. So, just
       #make it build a few smoke packages.
    runs-on: ubuntu-22.04
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: build some
      run: ./tool/go build ./ipn/... ./wgengine/ ./types/... ./control/controlclient
      env:
        GOOS: ios
        GOARCH: arm64

  android:
    # similar to cross above, but android fails to build a few pieces of the
    # repo. We should fix those pieces, they're small, but as a stepping stone,
    # only test the subset of android that our past smoke test checked.
    runs-on: ubuntu-22.04
    steps:
    - name: checkout
      uses: actions/checkout@v4
      # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed
      # and is only arm64. But it's a smoke build: it's not meant to catch everything. But it'll catch
      # some Android breakages early.
      # TODO(bradfitz): better; see https://github.com/tailscale/tailscale/issues/4482
    - name: build some
      run: ./tool/go install ./net/netns ./ipn/ipnlocal ./wgengine/magicsock/ ./wgengine/ ./wgengine/router/ ./wgengine/netstack ./util/dnsname/ ./ipn/ ./net/interfaces ./wgengine/router/ ./tailcfg/ ./types/logger/ ./net/dns ./hostinfo ./version
      env:
        GOOS: android
        GOARCH: arm64

  wasm: # builds tsconnect, which is the only wasm build we support
    runs-on: ubuntu-22.04
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: Restore Cache
      uses: actions/cache@v3
      with:
        # Note: unlike the other setups, this is only grabbing the mod download
        # cache, rather than the whole mod directory, as the download cache
        # contains zips that can be unpacked in parallel faster than they can be
        # fetched and extracted by tar
        path: |
          ~/.cache/go-build
          ~/go/pkg/mod/cache
          ~\AppData\Local\go-build          
        # The -2- here should be incremented when the scheme of data to be
        # cached changes (e.g. path above changes).
        key: ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }}
        restore-keys: |
          ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }}
          ${{ github.job }}-${{ runner.os }}-go-2-          
    - name: build tsconnect client
      run: ./tool/go build ./cmd/tsconnect/wasm ./cmd/tailscale/cli
      env:
        GOOS: js
        GOARCH: wasm
    - name: build tsconnect server
      # Note, no GOOS/GOARCH in env on this build step, we're running a build
      # tool that handles the build itself.
      run: |
        ./tool/go run ./cmd/tsconnect --fast-compression build
        ./tool/go run ./cmd/tsconnect --fast-compression build-pkg        

  tailscale_go: # Subset of tests that depend on our custom Go toolchain.
    runs-on: ubuntu-22.04
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: test tailscale_go
      run: ./tool/go test -tags=tailscale_go,ts_enable_sockstats ./net/sockstats/...


  fuzz:
    # This target periodically breaks (see TS_FUZZ_CURRENTLY_BROKEN at the top
    # of the file), so it's more complex than usual: the 'build fuzzers' step
    # might fail, and depending on the value of 'TS_FUZZ_CURRENTLY_BROKEN', that
    # might or might not be fine. The steps after the build figure out whether
    # the success/failure is expected, and appropriately pass/fail the job
    # overall accordingly.
    #
    # Practically, this means that all steps after 'build fuzzers' must have an
    # explicit 'if' condition, because the default condition for steps is
    # 'success()', meaning "only run this if no previous steps failed".
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-22.04
    steps:
    - name: build fuzzers
      id: build
      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
      # continue-on-error makes steps.build.conclusion be 'success' even if
      # steps.build.outcome is 'failure'. This means this step does not
      # contribute to the job's overall pass/fail evaluation.
      continue-on-error: true
      with:
       oss-fuzz-project-name: 'tailscale'
       dry-run: false
       language: go
    - name: report unexpectedly broken fuzz build
      if: steps.build.outcome == 'failure' && env.TS_FUZZ_CURRENTLY_BROKEN != 'true'
      run: |
        echo "fuzzer build failed, see above for why"
        echo "if the failure is due to OSS-Fuzz not being on the latest Go yet,"
        echo "set TS_FUZZ_CURRENTLY_BROKEN=true in .github/workflows/test.yml"
        echo "to temporarily disable fuzzing until OSS-Fuzz works again."
        exit 1        
    - name: report unexpectedly working fuzz build
      if: steps.build.outcome == 'success' && env.TS_FUZZ_CURRENTLY_BROKEN == 'true'
      run: |
        echo "fuzzer build succeeded, but we expect it to be broken"
        echo "please set TS_FUZZ_CURRENTLY_BROKEN=false in .github/workflows/test.yml"
        echo "to reenable fuzz testing"
        exit 1        
    - name: run fuzzers
      id: run
      # Run the fuzzers whenever they're able to build, even if we're going to
      # report a failure because TS_FUZZ_CURRENTLY_BROKEN is set to the wrong
      # value.
      if: steps.build.outcome == 'success'
      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
      with:
        oss-fuzz-project-name: 'tailscale'
        fuzz-seconds: 300
        dry-run: false
        language: go
    - name: upload crash
      uses: actions/upload-artifact@v3
      if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
      with:
        name: artifacts
        path: ./out/artifacts

  depaware:
    runs-on: ubuntu-22.04
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: check depaware
      run: |
        export PATH=$(./tool/go env GOROOT)/bin:$PATH
        find . -name 'depaware.txt' | xargs -n1 dirname | xargs ./tool/go run github.com/tailscale/depaware --check        

  go_generate:
    runs-on: ubuntu-22.04
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: check that 'go generate' is clean
      run: |
        pkgs=$(./tool/go list ./... | grep -v dnsfallback)
        ./tool/go generate $pkgs
        echo
        echo
        git diff --name-only --exit-code || (echo "The files above need updating. Please run 'go generate'."; exit 1)        

  go_mod_tidy:
    runs-on: ubuntu-22.04
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: check that 'go mod tidy' is clean
      run: |
        ./tool/go mod tidy
        echo
        echo
        git diff --name-only --exit-code || (echo "Please run 'go mod tidy'."; exit 1)        

  licenses:
    runs-on: ubuntu-22.04
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: check licenses
      run: ./scripts/check_license_headers.sh .

  staticcheck:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false # don't abort the entire matrix if one element fails
      matrix:
        goos: ["linux", "windows", "darwin"]
        goarch: ["amd64"]
        include:
          - goos: "windows"
            goarch: "386"
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: install staticcheck
      run: GOBIN=~/.local/bin ./tool/go install honnef.co/go/tools/cmd/staticcheck
    - name: run staticcheck
      run: |
        export GOROOT=$(./tool/go env GOROOT)
        export PATH=$GOROOT/bin:$PATH
        staticcheck -- $(./tool/go list ./... | grep -v tempfork)        
      env:
        GOOS: ${{ matrix.goos }}
        GOARCH: ${{ matrix.goarch }}

  notify_slack:
    if: always()
    # Any of these jobs failing causes a slack notification.
    needs: 
      - android
      - test
      - windows
      - vm
      - cross
      - ios
      - wasm
      - tailscale_go
      - fuzz
      - depaware
      - go_generate
      - go_mod_tidy
      - licenses
      - staticcheck
    runs-on: ubuntu-22.04
    steps:
    - name: notify  
      # Only notify slack for merged commits, not PR failures.
      #
      # It may be tempting to move this condition into the job's 'if' block, but
      # don't: Github only collapses the test list into "everything is OK" if
      # all jobs succeeded. A skipped job results in the list staying expanded.
      # By having the job always run, but skipping its only step as needed, we
      # let the CI output collapse nicely in PRs.
      if: failure() && github.event_name == 'push'
      uses: ruby/action-slack@v3.2.1
      with:
        payload: |
          {
            "attachments": [{
              "title": "Failure: ${{ github.workflow }}",
              "title_link": "https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks",         
              "text": "${{ github.repository }}@${{ github.ref_name }}: <https://github.com/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>",
              "fields": [{ "value": ${{ toJson(github.event.head_commit.message) }}, "short": false }],
              "footer": "${{ github.event.head_commit.committer.name }} at ${{ github.event.head_commit.timestamp }}",
              "color": "danger"
            }]
          }          
      env:
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

  check_mergeability:
    if: always()
    runs-on: ubuntu-22.04
    needs:
      - android
      - test
      - windows
      - vm
      - cross
      - ios
      - wasm
      - tailscale_go
      - fuzz
      - depaware
      - go_generate
      - go_mod_tidy
      - licenses
      - staticcheck
    steps:
    - name: Decide if change is okay to merge
      if: github.event_name != 'push'
      uses: re-actors/alls-green@release/v1
      with:
        jobs: ${{ toJSON(needs) }}