From 195300f56e6cf34b0bc6139cb5342bcf40479561 Mon Sep 17 00:00:00 2001 From: Stanislav Chzhen Date: Mon, 3 Apr 2023 18:22:15 +0300 Subject: [PATCH] Pull request 1778: AG-20200-translation-script-update-auto Merge in DNS/adguard-home from AG-20200-translation-script-update-auto to master Squashed commit of the following: commit 22c68f8443c59a5ba7ff7cd33c395f6dcf321e04 Author: Stanislav Chzhen Date: Mon Apr 3 18:08:40 2023 +0300 scripts: imp err more commit a6ea94b75c4bb09868f45f5c61d62622acf36d0c Merge: 69749b17 2a0d0629 Author: Stanislav Chzhen Date: Mon Apr 3 17:38:05 2023 +0300 Merge branch 'master' into AG-20200-translation-script-update-auto commit 69749b1767bd49d27c96ac17ef049f6ff6827f86 Author: Stanislav Chzhen Date: Mon Apr 3 17:37:27 2023 +0300 scripts: imp err commit 6d3eb3270e6fcbe94e9c8f73d654b65a15abcb37 Author: Stanislav Chzhen Date: Mon Apr 3 13:10:08 2023 +0300 scripts: imp err msg commit a95e3383f1c27b73eaa570dbe8e008c2bdf22be5 Merge: 16caba76 3575aa05 Author: Stanislav Chzhen Date: Mon Apr 3 12:06:28 2023 +0300 Merge branch 'master' into AG-20200-translation-script-update-auto commit 16caba76f0a16d70542f6fa0d6d83b134c630da4 Author: Stanislav Chzhen Date: Mon Apr 3 12:05:49 2023 +0300 scripts: fix err commit 3566193a7db677420722938c98089a40809c8739 Merge: 55efdeb8 da9008ab Author: Stanislav Chzhen Date: Thu Mar 30 13:13:54 2023 +0300 Merge branch 'master' into AG-20200-translation-script-update-auto commit 55efdeb80b44183767b188109f1e21aac2fa9839 Author: Stanislav Chzhen Date: Thu Mar 30 13:13:05 2023 +0300 scripts: simplify commit 4a090a6f015e4adb9d5a3f6e3c3c5294daf67f1d Merge: 571b2a29 c576d505 Author: Stanislav Chzhen Date: Wed Mar 29 14:10:06 2023 +0300 Merge branch 'master' into AG-20200-translation-script-update-auto commit 571b2a29777e694971cc02c895328d733b411803 Author: Stanislav Chzhen Date: Wed Mar 29 14:09:17 2023 +0300 scripts: fix log msg commit 6e92a76c4b9b1240501612878d5f42b3058cba32 Merge: 207c8bac 487675b9 Author: Stanislav Chzhen Date: Tue Mar 28 18:01:58 2023 +0300 Merge branch 'master' into AG-20200-translation-script-update-auto commit 207c8bacd4818c496b409cee96a4b6f65fbf8c24 Author: Stanislav Chzhen Date: Tue Mar 28 18:01:23 2023 +0300 scripts: add verbose flag commit e82270f53ce5cf8b1fdb39bff6367fd800483abb Merge: 11761bdc 132ec556 Author: Stanislav Chzhen Date: Mon Mar 27 15:09:21 2023 +0300 Merge branch 'master' into AG-20200-translation-script-update-auto commit 11761bdc3d9fd10221d0c21d993db0983d9e222f Author: Stanislav Chzhen Date: Mon Mar 27 15:08:39 2023 +0300 scripts: upd readme commit cdac6cf37022e67d4a45be587210765294e96270 Merge: 5f824358 df61741f Author: Stanislav Chzhen Date: Thu Mar 23 16:28:37 2023 +0300 Merge branch 'master' into AG-20200-translation-script-update-auto commit 5f82435847d74bf12a7b450b70d9326a57c99da6 Author: Stanislav Chzhen Date: Thu Mar 23 16:27:01 2023 +0300 scripts: add locale update auto --- scripts/README.md | 3 + scripts/translations/main.go | 150 ++++++++++++++++++++++++++++------- 2 files changed, 126 insertions(+), 27 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 4821849f..cc823a99 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -189,6 +189,9 @@ manifest file templates, and helper scripts. * `go run main.go unused`: show the list of unused strings. + * `go run main.go auto-add`: add locales with additions to the git and + restore locales with deletions. + After the download you'll find the output locales in the `client/src/__locales/` directory. diff --git a/scripts/translations/main.go b/scripts/translations/main.go index 7afd7666..a555be19 100644 --- a/scripts/translations/main.go +++ b/scripts/translations/main.go @@ -3,6 +3,7 @@ package main import ( + "bufio" "bytes" "encoding/json" "flag" @@ -13,12 +14,14 @@ import ( "net/textproto" "net/url" "os" + "os/exec" "path/filepath" "strings" "sync" "time" "github.com/AdguardTeam/AdGuardHome/internal/aghio" + "github.com/AdguardTeam/AdGuardHome/internal/aghos" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" "golang.org/x/exp/maps" @@ -76,19 +79,19 @@ func main() { switch os.Args[1] { case "summary": err = summary(conf.Languages) - check(err) case "download": err = download(uri, projectID, conf.Languages) - check(err) case "unused": - err = unused() - check(err) + err = unused(conf.LocalizableFiles[0]) case "upload": err = upload(uri, projectID, conf.BaseLangcode) - check(err) + case "auto-add": + err = autoAdd(conf.LocalizableFiles[0]) default: usage("unknown command") } + + check(err) } // check is a simple error-checking helper for scripts. @@ -108,11 +111,14 @@ Commands: summary Print summary. download [-n ] - Download translations. count is a number of concurrent downloads. + Download translations. count is a number of concurrent downloads. unused Print unused strings. upload - Upload translations.` + Upload translations. + auto-add + Add locales with additions to the git and restore locales with + deletions.` if addStr != "" { fmt.Printf("%s\n%s\n", addStr, usageStr) @@ -135,6 +141,8 @@ type twoskyConf struct { // readTwoskyConf returns configuration. func readTwoskyConf() (t twoskyConf, err error) { + defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }() + b, err := os.ReadFile(twoskyConfFile) if err != nil { // Don't wrap the error since it's informative enough as is. @@ -163,6 +171,10 @@ func readTwoskyConf() (t twoskyConf, err error) { } } + if len(conf.LocalizableFiles) == 0 { + return twoskyConf{}, errors.Error("no localizable files specified") + } + return conf, nil } @@ -232,7 +244,7 @@ func download(uri *url.URL, projectID string, langs languages) (err error) { err = flagSet.Parse(os.Args[2:]) if err != nil { - // Don't wrap the error since there is exit on error. + // Don't wrap the error since it's informative enough as is. return err } @@ -246,12 +258,12 @@ func download(uri *url.URL, projectID string, langs languages) (err error) { Timeout: 10 * time.Second, } - var wg sync.WaitGroup + wg := &sync.WaitGroup{} uriCh := make(chan *url.URL, len(langs)) for i := 0; i < numWorker; i++ { wg.Add(1) - go downloadWorker(&wg, client, uriCh) + go downloadWorker(wg, client, uriCh) } for lang := range langs { @@ -342,9 +354,7 @@ func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) } // unused prints unused text labels. -func unused() (err error) { - fileNames := []string{} - basePath := filepath.Join(localesDir, defaultBaseFile) +func unused(basePath string) (err error) { baseLoc, err := readLocales(basePath) if err != nil { return fmt.Errorf("unused: %w", err) @@ -352,9 +362,10 @@ func unused() (err error) { locDir := filepath.Clean(localesDir) + fileNames := []string{} err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error { if err != nil { - log.Info("accessing a path %q: %s", name, err) + log.Info("warning: accessing a path %q: %s", name, err) return nil } @@ -379,12 +390,11 @@ func unused() (err error) { return fmt.Errorf("filepath walking %q: %w", srcDir, err) } - err = removeUnused(fileNames, baseLoc) - - return errors.Annotate(err, "removing unused: %w") + return findUnused(fileNames, baseLoc) } -func removeUnused(fileNames []string, loc locales) (err error) { +// findUnused prints unused text labels from fileNames. +func findUnused(fileNames []string, loc locales) (err error) { knownUsed := []textLabel{ "blocking_mode_refused", "blocking_mode_nxdomain", @@ -399,8 +409,7 @@ func removeUnused(fileNames []string, loc locales) (err error) { var buf []byte buf, err = os.ReadFile(fn) if err != nil { - // Don't wrap the error since it's informative enough as is. - return err + return fmt.Errorf("finding unused: %w", err) } for k := range loc { @@ -410,19 +419,14 @@ func removeUnused(fileNames []string, loc locales) (err error) { } } - printUnused(loc) - - return nil -} - -// printUnused text labels to stdout. -func printUnused(loc locales) { keys := maps.Keys(loc) slices.Sort(keys) for _, v := range keys { fmt.Println(v) } + + return nil } // upload base translation. uri is the base URL. projectID is the name of the @@ -536,3 +540,95 @@ func send(uriStr, cType string, buf *bytes.Buffer) (err error) { return nil } + +// autoAdd adds locales with additions to the git and restores locales with +// deletions. +func autoAdd(basePath string) (err error) { + defer func() { err = errors.Annotate(err, "auto add: %w") }() + + adds, dels, err := changedLocales() + if err != nil { + // Don't wrap the error since it's informative enough as is. + return err + } + + if slices.Contains(dels, basePath) { + return errors.Error("base locale contains deletions") + } + + var ( + args []string + code int + out []byte + ) + + if len(adds) > 0 { + args = append([]string{"add"}, adds...) + code, out, err = aghos.RunCommand("git", args...) + + if err != nil || code != 0 { + return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err) + } + } + + if len(dels) > 0 { + args = append([]string{"restore"}, dels...) + code, out, err = aghos.RunCommand("git", args...) + + if err != nil || code != 0 { + return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err) + } + } + + return nil +} + +// changedLocales returns cleaned paths of locales with changes or error. adds +// is the list of locales with only additions. dels is the list of locales +// with only deletions. +func changedLocales() (adds, dels []string, err error) { + defer func() { err = errors.Annotate(err, "getting changes: %w") }() + + cmd := exec.Command("git", "diff", "--numstat", localesDir) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, fmt.Errorf("piping: %w", err) + } + + err = cmd.Start() + if err != nil { + return nil, nil, fmt.Errorf("starting: %w", err) + } + + scanner := bufio.NewScanner(stdout) + + for scanner.Scan() { + line := scanner.Text() + + fields := strings.Fields(line) + if len(fields) < 3 { + return nil, nil, fmt.Errorf("invalid input: %q", line) + } + + path := fields[2] + + if fields[0] == "0" { + dels = append(dels, path) + } else if fields[1] == "0" { + adds = append(adds, path) + } + } + + err = scanner.Err() + if err != nil { + return nil, nil, fmt.Errorf("scanning: %w", err) + } + + err = cmd.Wait() + if err != nil { + return nil, nil, fmt.Errorf("waiting: %w", err) + } + + return adds, dels, nil +}