From 3d7dcc286e46b14eace27ae179a8dd4a6d1d5b2c Mon Sep 17 00:00:00 2001 From: Max Andreev Date: Fri, 28 Apr 2023 14:33:39 +0300 Subject: [PATCH] Add indexer (#32) * Add indexer * Add python and get_env.py * Add submodules: recursive * Fix build.yml * fix upload * Upgrade nodejs action --- .github/workflows/build.yml | 97 +++++++++++++----------------- scripts/get_env.py | 116 ++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 54 deletions(-) create mode 100644 scripts/get_env.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 141bbc1..7318077 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,6 @@ name: 'Build' -on: +on: push: branches: - dev @@ -13,78 +13,68 @@ jobs: build: runs-on: ubuntu-22.04 steps: - - name: Store UID + - name: 'Store UID' id: uid run: | - id=`id -u $USER` - echo ::set-output name=id::$id + echo id="$(id -u $USER)" >> $GITHUB_OUTPUT - - name: Cleanup workspace - uses: AutoModality/action-clean@v1 + - name: 'Wipe workspace' + run: find ./ -mount -maxdepth 1 -exec rm -rf {} \; - - name: Decontaminate previous build leftovers - run: | - if [ -d .git ] - then - git submodule status \ - || git checkout `git rev-list --max-parents=0 HEAD | tail -n 1` - fi - - - name: Checkout code - uses: actions/checkout@v2 + - name: 'Checkout code' + uses: actions/checkout@v3 with: - submodules: recursive ref: ${{ github.event.pull_request.head.sha }} + submodules: recursive - - name: Generate suffix and folder name + - name: 'Setup python' + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: 'Get commit details' id: names run: | - REF=${{ github.ref }} if [[ ${{ github.event_name }} == 'pull_request' ]]; then - REF=${{ github.head_ref }} - fi - BRANCH_OR_TAG=${REF#refs/*/} - SHA=$(git rev-parse --short HEAD) - - if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then - SUFFIX=${BRANCH_OR_TAG//\//_} + TYPE="pull" + elif [[ "${{ github.ref }}" == "refs/tags/"* ]]; then + TYPE="tag" else - SUFFIX=${BRANCH_OR_TAG//\//_}-$(date +'%d%m%Y')-${SHA} + TYPE="other" fi + python3 scripts/get_env.py "--event_file=${{ github.event_path }}" "--type=$TYPE" + echo "event_type=$TYPE" >> $GITHUB_OUTPUT - echo "::set-output name=artifacts-path::${BRANCH_OR_TAG}" - echo "::set-output name=suffix::${SUFFIX}" - - - name: Setup node - uses: actions/setup-node@v2 + - name: 'Setup node' + uses: actions/setup-node@v3 with: node-version: '17' cache: 'npm' cache-dependency-path: components/svelte-portal - - name: Install node dependencies + - name: 'Install node dependencies' run: npm install working-directory: components/svelte-portal - - name: Build captive portal + - name: 'Build captive portal' run: npm run build working-directory: components/svelte-portal - - name: Build firmware + - name: 'Build firmware' uses: Intake-Health/esp-idf-ci-action@release-v4.4 - - name: Fix ownership + - name: 'Fix ownership' uses: peter-murray/reset-workspace-ownership-action@v1 with: user_id: ${{ steps.uid.outputs.id }} - - name: Make artifacts directory + - name: 'Make artifacts directory' if: ${{ !github.event.pull_request.head.repo.fork }} run: | - test -d artifacts && rm -rf artifacts || true + rm -rf artifacts mkdir artifacts - - name: Move upload files + - name: 'Move upload files' if: ${{ !github.event.pull_request.head.repo.fork }} run: | mv build/bootloader/bootloader.bin artifacts/ @@ -92,25 +82,24 @@ jobs: mv build/blackmagic.bin artifacts/ cp scripts/flash_nix.sh artifacts/ - - name: Generate flash.command file + - name: 'Generate flash.command file' if: ${{ !github.event.pull_request.head.repo.fork }} run: | - echo "esptool.py -p (PORT) -b 460800 --before default_reset --after hard_reset --chip esp32s2 write_flash $(tr '\n' ' ' < build/flash_args)" > artifacts/flash.command + echo "esptool.py -p (PORT) -b 460800 --before default_reset --after hard_reset --chip esp32s2 write_flash $(tr '\n' ' ' < build/flash_args)" > artifacts/flash.command sed -i 's/partition_table\///g' artifacts/flash.command sed -i 's/bootloader\///g' artifacts/flash.command - - name: Upload artifacts to update server + - name: 'Generate archive' if: ${{ !github.event.pull_request.head.repo.fork }} - uses: burnett01/rsync-deployments@5.1 - with: - switches: -avzP --delete --mkpath - path: artifacts/ - remote_path: ${{ secrets.RSYNC_DEPLOY_BASE_PATH }}${{steps.names.outputs.artifacts-path}}/ - remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }} - remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }} - remote_user: ${{ secrets.RSYNC_DEPLOY_USER }} - remote_key: ${{ secrets.RSYNC_DEPLOY_KEY }} - - - name: Cleanup artifacts run: | - rm -rf artifacts \ No newline at end of file + mkdir -p "artifacts-archive" + tar -cvf artifacts-archive/blackmagic-firmware-s2-full-${SUFFIX}.tgz -C artifacts/ . + + - name: 'Upload artifacts to update server' + if: ${{ !github.event.pull_request.head.repo.fork }} + run: | + FILES=$(for CUR in $(ls artifacts-archive/); do echo "-F files=@artifacts-archive/$CUR"; done) + curl --fail -L -H "Token: ${{ secrets.INDEXER_TOKEN }}" \ + -F "branch=${BRANCH_NAME}" \ + ${FILES[@]} \ + "${{ secrets.INDEXER_UPLOAD_URL }}" diff --git a/scripts/get_env.py b/scripts/get_env.py new file mode 100644 index 0000000..92f9243 --- /dev/null +++ b/scripts/get_env.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +import ssl +import json +import os +import shlex +import re +import string +import random +import argparse +import datetime +import urllib.request + + +def id_gen(size=5, chars=string.ascii_uppercase + string.digits): + return "".join(random.choice(chars) for _ in range(size)) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--event_file", help="Current GitHub event file", required=True) + parser.add_argument( + "--type", + help="Event file type", + required=True, + choices=["pull", "tag", "other"], + ) + args = parser.parse_args() + return args + + +def get_commit_json(event): + context = ssl._create_unverified_context() + commit_url = event["pull_request"]["base"]["repo"]["commits_url"].replace( + "{/sha}", f"/{event['pull_request']['head']['sha']}" + ) + with urllib.request.urlopen(commit_url, context=context) as commit_file: + commit_json = json.loads(commit_file.read().decode("utf-8")) + return commit_json + + +def get_details(event, args): + data = {} + current_time = datetime.datetime.utcnow().date() + if args.type == "pull": + commit_json = get_commit_json(event) + data["commit_comment"] = shlex.quote(commit_json["commit"]["message"]) + data["commit_hash"] = commit_json["sha"] + ref = event["pull_request"]["head"]["ref"] + data["pull_id"] = event["pull_request"]["number"] + data["pull_name"] = shlex.quote(event["pull_request"]["title"]) + elif args.type == "tag": + data["commit_comment"] = shlex.quote(event["head_commit"]["message"]) + data["commit_hash"] = event["head_commit"]["id"] + ref = event["ref"] + else: + data["commit_comment"] = shlex.quote(event["commits"][-1]["message"]) + data["commit_hash"] = event["commits"][-1]["id"] + ref = event["ref"] + data["commit_sha"] = data["commit_hash"][:8] + data["branch_name"] = re.sub("refs/\w+/", "", ref) + data["suffix"] = ( + data["branch_name"].replace("/", "_") + + "-" + + current_time.strftime("%d%m%Y") + + "-" + + data["commit_sha"] + ) + if ref.startswith("refs/tags/"): + data["suffix"] = data["branch_name"].replace("/", "_") + return data + + +def add_env(name, value, file): + delimeter = id_gen() + print(f"{name}<<{delimeter}", file=file) + print(f"{value}", file=file) + print(f"{delimeter}", file=file) + + +def add_set_output_var(name, value, file): + print(f"{name}={value}", file=file) + + +def add_envs(data, gh_env_file, gh_out_file, args): + add_env("COMMIT_MSG", data["commit_comment"], gh_env_file) + add_env("COMMIT_HASH", data["commit_hash"], gh_env_file) + add_env("COMMIT_SHA", data["commit_sha"], gh_env_file) + add_env("SUFFIX", data["suffix"], gh_env_file) + add_env("BRANCH_NAME", data["branch_name"], gh_env_file) + add_env("DIST_SUFFIX", data["suffix"], gh_env_file) + add_env("WORKFLOW_BRANCH_OR_TAG", data["branch_name"], gh_env_file) + add_set_output_var("branch_name", data["branch_name"], gh_out_file) + add_set_output_var("commit_sha", data["commit_sha"], gh_out_file) + add_set_output_var("default_target", os.getenv("DEFAULT_TARGET"), gh_out_file) + add_set_output_var("suffix", data["suffix"], gh_out_file) + if args.type == "pull": + add_env("PULL_ID", data["pull_id"], gh_env_file) + add_env("PULL_NAME", data["pull_name"], gh_env_file) + + +def main(): + args = parse_args() + event_file = open(args.event_file, "r") + event = json.load(event_file) + gh_env_file = open(os.environ["GITHUB_ENV"], "a") + gh_out_file = open(os.environ["GITHUB_OUTPUT"], "a") + data = get_details(event, args) + add_envs(data, gh_env_file, gh_out_file, args) + event_file.close() + gh_env_file.close() + gh_out_file.close() + + +if __name__ == "__main__": + main()