Add indexer (#32)

* Add indexer
* Add python and get_env.py
* Add submodules: recursive
* Fix build.yml
* fix upload
* Upgrade nodejs action

.github/workflows/build.yml

@@ -13,78 +13,68 @@ jobs:
   build:
     runs-on: ubuntu-22.04
     steps:
-      - name: Store UID
+      - name: 'Store UID'
         id: uid
         run: |
-          id=`id -u $USER`
-          echo ::set-output name=id::$id
+          echo id="$(id -u $USER)" >> $GITHUB_OUTPUT
 
-      - name: Cleanup workspace
-        uses: AutoModality/action-clean@v1
+      - name: 'Wipe workspace'
+        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
 
-      - name: Decontaminate previous build leftovers
-        run: |
-          if [ -d .git ]
-          then
-            git submodule status \
-              || git checkout `git rev-list --max-parents=0 HEAD | tail -n 1`
-          fi
-
-      - name: Checkout code
-        uses: actions/checkout@v2
+      - name: 'Checkout code'
+        uses: actions/checkout@v3
         with:
-          submodules: recursive
           ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
 
-      - name: Generate suffix and folder name
+      - name: 'Setup python'
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: 'Get commit details'
         id: names
         run: |
-          REF=${{ github.ref }}
           if [[ ${{ github.event_name }} == 'pull_request' ]]; then
-            REF=${{ github.head_ref }}
-          fi
-          BRANCH_OR_TAG=${REF#refs/*/}
-          SHA=$(git rev-parse --short HEAD)
-          
-          if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
-            SUFFIX=${BRANCH_OR_TAG//\//_}
+            TYPE="pull"
+          elif [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
+            TYPE="tag"
           else
-            SUFFIX=${BRANCH_OR_TAG//\//_}-$(date +'%d%m%Y')-${SHA}
+            TYPE="other"
           fi
+          python3 scripts/get_env.py "--event_file=${{ github.event_path }}" "--type=$TYPE"
+          echo "event_type=$TYPE" >> $GITHUB_OUTPUT
 
-          echo "::set-output name=artifacts-path::${BRANCH_OR_TAG}"
-          echo "::set-output name=suffix::${SUFFIX}"
-
-      - name: Setup node
-        uses: actions/setup-node@v2
+      - name: 'Setup node'
+        uses: actions/setup-node@v3
         with:
           node-version: '17'
           cache: 'npm'
           cache-dependency-path: components/svelte-portal
 
-      - name: Install node dependencies
+      - name: 'Install node dependencies'
         run: npm install
         working-directory: components/svelte-portal
 
-      - name: Build captive portal
+      - name: 'Build captive portal'
         run: npm run build
         working-directory: components/svelte-portal
 
-      - name: Build firmware
+      - name: 'Build firmware'
         uses: Intake-Health/esp-idf-ci-action@release-v4.4
 
-      - name: Fix ownership
+      - name: 'Fix ownership'
         uses: peter-murray/reset-workspace-ownership-action@v1
         with:
           user_id: ${{ steps.uid.outputs.id }}
 
-      - name: Make artifacts directory
+      - name: 'Make artifacts directory'
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
-          test -d artifacts && rm -rf artifacts || true
+          rm -rf artifacts
           mkdir artifacts
 
-      - name: Move upload files
+      - name: 'Move upload files'
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
           mv build/bootloader/bootloader.bin artifacts/
@@ -92,25 +82,24 @@ jobs:
           mv build/blackmagic.bin artifacts/
           cp scripts/flash_nix.sh artifacts/
 
-      - name: Generate flash.command file
+      - name: 'Generate flash.command file'
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
-          echo "esptool.py -p (PORT) -b 460800 --before default_reset --after hard_reset --chip esp32s2  write_flash $(tr '\n' ' ' < build/flash_args)" > artifacts/flash.command
+          echo "esptool.py -p (PORT) -b 460800 --before default_reset --after hard_reset --chip esp32s2 write_flash $(tr '\n' ' ' < build/flash_args)" > artifacts/flash.command
           sed -i 's/partition_table\///g' artifacts/flash.command
           sed -i 's/bootloader\///g' artifacts/flash.command
 
-      - name: Upload artifacts to update server
+      - name: 'Generate archive'
         if: ${{ !github.event.pull_request.head.repo.fork }}
-        uses: burnett01/rsync-deployments@5.1
-        with:
-          switches: -avzP --delete --mkpath
-          path: artifacts/
-          remote_path: ${{ secrets.RSYNC_DEPLOY_BASE_PATH }}${{steps.names.outputs.artifacts-path}}/
-          remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }}
-          remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }}
-          remote_user: ${{ secrets.RSYNC_DEPLOY_USER }}
-          remote_key: ${{ secrets.RSYNC_DEPLOY_KEY }}
-
-      - name: Cleanup artifacts
         run: |
-          rm -rf artifacts
+          mkdir -p "artifacts-archive"
+          tar -cvf artifacts-archive/blackmagic-firmware-s2-full-${SUFFIX}.tgz -C artifacts/ .
+
+      - name: 'Upload artifacts to update server'
+        if: ${{ !github.event.pull_request.head.repo.fork }}
+        run: |
+          FILES=$(for CUR in $(ls artifacts-archive/); do echo "-F files=@artifacts-archive/$CUR"; done)
+          curl --fail -L -H "Token: ${{ secrets.INDEXER_TOKEN }}" \
+              -F "branch=${BRANCH_NAME}" \
+              ${FILES[@]} \
+              "${{ secrets.INDEXER_UPLOAD_URL }}"

scripts/get_env.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+import ssl
+import json
+import os
+import shlex
+import re
+import string
+import random
+import argparse
+import datetime
+import urllib.request
+
+
+def id_gen(size=5, chars=string.ascii_uppercase + string.digits):
+    return "".join(random.choice(chars) for _ in range(size))
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--event_file", help="Current GitHub event file", required=True)
+    parser.add_argument(
+        "--type",
+        help="Event file type",
+        required=True,
+        choices=["pull", "tag", "other"],
+    )
+    args = parser.parse_args()
+    return args
+
+
+def get_commit_json(event):
+    context = ssl._create_unverified_context()
+    commit_url = event["pull_request"]["base"]["repo"]["commits_url"].replace(
+        "{/sha}", f"/{event['pull_request']['head']['sha']}"
+    )
+    with urllib.request.urlopen(commit_url, context=context) as commit_file:
+        commit_json = json.loads(commit_file.read().decode("utf-8"))
+    return commit_json
+
+
+def get_details(event, args):
+    data = {}
+    current_time = datetime.datetime.utcnow().date()
+    if args.type == "pull":
+        commit_json = get_commit_json(event)
+        data["commit_comment"] = shlex.quote(commit_json["commit"]["message"])
+        data["commit_hash"] = commit_json["sha"]
+        ref = event["pull_request"]["head"]["ref"]
+        data["pull_id"] = event["pull_request"]["number"]
+        data["pull_name"] = shlex.quote(event["pull_request"]["title"])
+    elif args.type == "tag":
+        data["commit_comment"] = shlex.quote(event["head_commit"]["message"])
+        data["commit_hash"] = event["head_commit"]["id"]
+        ref = event["ref"]
+    else:
+        data["commit_comment"] = shlex.quote(event["commits"][-1]["message"])
+        data["commit_hash"] = event["commits"][-1]["id"]
+        ref = event["ref"]
+    data["commit_sha"] = data["commit_hash"][:8]
+    data["branch_name"] = re.sub("refs/\w+/", "", ref)
+    data["suffix"] = (
+        data["branch_name"].replace("/", "_")
+        + "-"
+        + current_time.strftime("%d%m%Y")
+        + "-"
+        + data["commit_sha"]
+    )
+    if ref.startswith("refs/tags/"):
+        data["suffix"] = data["branch_name"].replace("/", "_")
+    return data
+
+
+def add_env(name, value, file):
+    delimeter = id_gen()
+    print(f"{name}<<{delimeter}", file=file)
+    print(f"{value}", file=file)
+    print(f"{delimeter}", file=file)
+
+
+def add_set_output_var(name, value, file):
+    print(f"{name}={value}", file=file)
+
+
+def add_envs(data, gh_env_file, gh_out_file, args):
+    add_env("COMMIT_MSG", data["commit_comment"], gh_env_file)
+    add_env("COMMIT_HASH", data["commit_hash"], gh_env_file)
+    add_env("COMMIT_SHA", data["commit_sha"], gh_env_file)
+    add_env("SUFFIX", data["suffix"], gh_env_file)
+    add_env("BRANCH_NAME", data["branch_name"], gh_env_file)
+    add_env("DIST_SUFFIX", data["suffix"], gh_env_file)
+    add_env("WORKFLOW_BRANCH_OR_TAG", data["branch_name"], gh_env_file)
+    add_set_output_var("branch_name", data["branch_name"], gh_out_file)
+    add_set_output_var("commit_sha", data["commit_sha"], gh_out_file)
+    add_set_output_var("default_target", os.getenv("DEFAULT_TARGET"), gh_out_file)
+    add_set_output_var("suffix", data["suffix"], gh_out_file)
+    if args.type == "pull":
+        add_env("PULL_ID", data["pull_id"], gh_env_file)
+        add_env("PULL_NAME", data["pull_name"], gh_env_file)
+
+
+def main():
+    args = parse_args()
+    event_file = open(args.event_file, "r")
+    event = json.load(event_file)
+    gh_env_file = open(os.environ["GITHUB_ENV"], "a")
+    gh_out_file = open(os.environ["GITHUB_OUTPUT"], "a")
+    data = get_details(event, args)
+    add_envs(data, gh_env_file, gh_out_file, args)
+    event_file.close()
+    gh_env_file.close()
+    gh_out_file.close()
+
+
+if __name__ == "__main__":
+    main()