Pull request 1778: AG-20200-translation-script-update-auto

Merge in DNS/adguard-home from AG-20200-translation-script-update-auto to master

Squashed commit of the following:

commit 22c68f8443c59a5ba7ff7cd33c395f6dcf321e04
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 18:08:40 2023 +0300

    scripts: imp err more

commit a6ea94b75c4bb09868f45f5c61d62622acf36d0c
Merge: 69749b17 2a0d0629
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 17:38:05 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 69749b1767bd49d27c96ac17ef049f6ff6827f86
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 17:37:27 2023 +0300

    scripts: imp err

commit 6d3eb3270e6fcbe94e9c8f73d654b65a15abcb37
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 13:10:08 2023 +0300

    scripts: imp err msg

commit a95e3383f1c27b73eaa570dbe8e008c2bdf22be5
Merge: 16caba76 3575aa05
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 12:06:28 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 16caba76f0a16d70542f6fa0d6d83b134c630da4
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 12:05:49 2023 +0300

    scripts: fix err

commit 3566193a7db677420722938c98089a40809c8739
Merge: 55efdeb8 da9008ab
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 30 13:13:54 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 55efdeb80b44183767b188109f1e21aac2fa9839
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 30 13:13:05 2023 +0300

    scripts: simplify

commit 4a090a6f015e4adb9d5a3f6e3c3c5294daf67f1d
Merge: 571b2a29 c576d505
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 29 14:10:06 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 571b2a29777e694971cc02c895328d733b411803
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 29 14:09:17 2023 +0300

    scripts: fix log msg

commit 6e92a76c4b9b1240501612878d5f42b3058cba32
Merge: 207c8bac 487675b9
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Mar 28 18:01:58 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 207c8bacd4818c496b409cee96a4b6f65fbf8c24
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Mar 28 18:01:23 2023 +0300

    scripts: add verbose flag

commit e82270f53ce5cf8b1fdb39bff6367fd800483abb
Merge: 11761bdc 132ec556
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 27 15:09:21 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 11761bdc3d9fd10221d0c21d993db0983d9e222f
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 27 15:08:39 2023 +0300

    scripts: upd readme

commit cdac6cf37022e67d4a45be587210765294e96270
Merge: 5f824358 df61741f
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 23 16:28:37 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 5f82435847d74bf12a7b450b70d9326a57c99da6
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 23 16:27:01 2023 +0300

    scripts: add locale update auto

scripts/README.md

@@ -189,6 +189,9 @@ manifest file templates, and helper scripts.
 
  *  `go run main.go unused`: show the list of unused strings.
 
+ *  `go run main.go auto-add`: add locales with additions to the git and
+    restore locales with deletions.
+
 After the download you'll find the output locales in the `client/src/__locales/`
 directory.
 

scripts/translations/main.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/json"
 	"flag"
@@ -13,12 +14,14 @@ import (
 	"net/textproto"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/AdguardTeam/AdGuardHome/internal/aghio"
+	"github.com/AdguardTeam/AdGuardHome/internal/aghos"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
 	"golang.org/x/exp/maps"
@@ -76,19 +79,19 @@ func main() {
 	switch os.Args[1] {
 	case "summary":
 		err = summary(conf.Languages)
-		check(err)
 	case "download":
 		err = download(uri, projectID, conf.Languages)
-		check(err)
 	case "unused":
-		err = unused()
+		err = unused(conf.LocalizableFiles[0])
-		check(err)
 	case "upload":
 		err = upload(uri, projectID, conf.BaseLangcode)
-		check(err)
+	case "auto-add":
+		err = autoAdd(conf.LocalizableFiles[0])
 	default:
 		usage("unknown command")
 	}
+
+	check(err)
 }
 
 // check is a simple error-checking helper for scripts.
@@ -112,7 +115,10 @@ Commands:
   unused
         Print unused strings.
   upload
-        Upload translations.`
+        Upload translations.
+  auto-add
+		Add locales with additions to the git and restore locales with
+		deletions.`
 
 	if addStr != "" {
 		fmt.Printf("%s\n%s\n", addStr, usageStr)
@@ -135,6 +141,8 @@ type twoskyConf struct {
 
 // readTwoskyConf returns configuration.
 func readTwoskyConf() (t twoskyConf, err error) {
+	defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }()
+
 	b, err := os.ReadFile(twoskyConfFile)
 	if err != nil {
 		// Don't wrap the error since it's informative enough as is.
@@ -163,6 +171,10 @@ func readTwoskyConf() (t twoskyConf, err error) {
 		}
 	}
 
+	if len(conf.LocalizableFiles) == 0 {
+		return twoskyConf{}, errors.Error("no localizable files specified")
+	}
+
 	return conf, nil
 }
 
@@ -232,7 +244,7 @@ func download(uri *url.URL, projectID string, langs languages) (err error) {
 
 	err = flagSet.Parse(os.Args[2:])
 	if err != nil {
-		// Don't wrap the error since there is exit on error.
+		// Don't wrap the error since it's informative enough as is.
 		return err
 	}
 
@@ -246,12 +258,12 @@ func download(uri *url.URL, projectID string, langs languages) (err error) {
 		Timeout: 10 * time.Second,
 	}
 
-	var wg sync.WaitGroup
+	wg := &sync.WaitGroup{}
 	uriCh := make(chan *url.URL, len(langs))
 
 	for i := 0; i < numWorker; i++ {
 		wg.Add(1)
-		go downloadWorker(&wg, client, uriCh)
+		go downloadWorker(wg, client, uriCh)
 	}
 
 	for lang := range langs {
@@ -342,9 +354,7 @@ func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode)
 }
 
 // unused prints unused text labels.
-func unused() (err error) {
+func unused(basePath string) (err error) {
-	fileNames := []string{}
-	basePath := filepath.Join(localesDir, defaultBaseFile)
 	baseLoc, err := readLocales(basePath)
 	if err != nil {
 		return fmt.Errorf("unused: %w", err)
@@ -352,9 +362,10 @@ func unused() (err error) {
 
 	locDir := filepath.Clean(localesDir)
 
+	fileNames := []string{}
 	err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error {
 		if err != nil {
-			log.Info("accessing a path %q: %s", name, err)
+			log.Info("warning: accessing a path %q: %s", name, err)
 
 			return nil
 		}
@@ -379,12 +390,11 @@ func unused() (err error) {
 		return fmt.Errorf("filepath walking %q: %w", srcDir, err)
 	}
 
-	err = removeUnused(fileNames, baseLoc)
+	return findUnused(fileNames, baseLoc)
-
-	return errors.Annotate(err, "removing unused: %w")
 }
 
-func removeUnused(fileNames []string, loc locales) (err error) {
+// findUnused prints unused text labels from fileNames.
+func findUnused(fileNames []string, loc locales) (err error) {
 	knownUsed := []textLabel{
 		"blocking_mode_refused",
 		"blocking_mode_nxdomain",
@@ -399,8 +409,7 @@ func removeUnused(fileNames []string, loc locales) (err error) {
 		var buf []byte
 		buf, err = os.ReadFile(fn)
 		if err != nil {
-			// Don't wrap the error since it's informative enough as is.
+			return fmt.Errorf("finding unused: %w", err)
-			return err
 		}
 
 		for k := range loc {
@@ -410,19 +419,14 @@ func removeUnused(fileNames []string, loc locales) (err error) {
 		}
 	}
 
-	printUnused(loc)
-
-	return nil
-}
-
-// printUnused text labels to stdout.
-func printUnused(loc locales) {
 	keys := maps.Keys(loc)
 	slices.Sort(keys)
 
 	for _, v := range keys {
 		fmt.Println(v)
 	}
+
+	return nil
 }
 
 // upload base translation.  uri is the base URL.  projectID is the name of the
@@ -536,3 +540,95 @@ func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
 
 	return nil
 }
+
+// autoAdd adds locales with additions to the git and restores locales with
+// deletions.
+func autoAdd(basePath string) (err error) {
+	defer func() { err = errors.Annotate(err, "auto add: %w") }()
+
+	adds, dels, err := changedLocales()
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return err
+	}
+
+	if slices.Contains(dels, basePath) {
+		return errors.Error("base locale contains deletions")
+	}
+
+	var (
+		args []string
+		code int
+		out  []byte
+	)
+
+	if len(adds) > 0 {
+		args = append([]string{"add"}, adds...)
+		code, out, err = aghos.RunCommand("git", args...)
+
+		if err != nil || code != 0 {
+			return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
+		}
+	}
+
+	if len(dels) > 0 {
+		args = append([]string{"restore"}, dels...)
+		code, out, err = aghos.RunCommand("git", args...)
+
+		if err != nil || code != 0 {
+			return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
+		}
+	}
+
+	return nil
+}
+
+// changedLocales returns cleaned paths of locales with changes or error.  adds
+// is the list of locales with only additions.  dels is the list of locales
+// with only deletions.
+func changedLocales() (adds, dels []string, err error) {
+	defer func() { err = errors.Annotate(err, "getting changes: %w") }()
+
+	cmd := exec.Command("git", "diff", "--numstat", localesDir)
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, nil, fmt.Errorf("piping: %w", err)
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return nil, nil, fmt.Errorf("starting: %w", err)
+	}
+
+	scanner := bufio.NewScanner(stdout)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		fields := strings.Fields(line)
+		if len(fields) < 3 {
+			return nil, nil, fmt.Errorf("invalid input: %q", line)
+		}
+
+		path := fields[2]
+
+		if fields[0] == "0" {
+			dels = append(dels, path)
+		} else if fields[1] == "0" {
+			adds = append(adds, path)
+		}
+	}
+
+	err = scanner.Err()
+	if err != nil {
+		return nil, nil, fmt.Errorf("scanning: %w", err)
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return nil, nil, fmt.Errorf("waiting: %w", err)
+	}
+
+	return adds, dels, nil
+}