mirrors
/
AdGuardHome
mirror of https://github.com/AdguardTeam/AdGuardHome.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pull request 1778: AG-20200-translation-script-update-auto 

Merge in DNS/adguard-home from AG-20200-translation-script-update-auto to master

Squashed commit of the following:

commit 22c68f8443c59a5ba7ff7cd33c395f6dcf321e04
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 18:08:40 2023 +0300

    scripts: imp err more

commit a6ea94b75c4bb09868f45f5c61d62622acf36d0c
Merge: 69749b17 2a0d0629
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 17:38:05 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 69749b1767bd49d27c96ac17ef049f6ff6827f86
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 17:37:27 2023 +0300

    scripts: imp err

commit 6d3eb3270e6fcbe94e9c8f73d654b65a15abcb37
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 13:10:08 2023 +0300

    scripts: imp err msg

commit a95e3383f1c27b73eaa570dbe8e008c2bdf22be5
Merge: 16caba76 3575aa05
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 12:06:28 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 16caba76f0a16d70542f6fa0d6d83b134c630da4
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 12:05:49 2023 +0300

    scripts: fix err

commit 3566193a7db677420722938c98089a40809c8739
Merge: 55efdeb8 da9008ab
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 30 13:13:54 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 55efdeb80b44183767b188109f1e21aac2fa9839
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 30 13:13:05 2023 +0300

    scripts: simplify

commit 4a090a6f015e4adb9d5a3f6e3c3c5294daf67f1d
Merge: 571b2a29 c576d505
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 29 14:10:06 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 571b2a29777e694971cc02c895328d733b411803
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 29 14:09:17 2023 +0300

    scripts: fix log msg

commit 6e92a76c4b9b1240501612878d5f42b3058cba32
Merge: 207c8bac 487675b9
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Mar 28 18:01:58 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 207c8bacd4818c496b409cee96a4b6f65fbf8c24
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Mar 28 18:01:23 2023 +0300

    scripts: add verbose flag

commit e82270f53ce5cf8b1fdb39bff6367fd800483abb
Merge: 11761bdc 132ec556
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 27 15:09:21 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 11761bdc3d9fd10221d0c21d993db0983d9e222f
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 27 15:08:39 2023 +0300

    scripts: upd readme

commit cdac6cf37022e67d4a45be587210765294e96270
Merge: 5f824358 df61741f
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 23 16:28:37 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 5f82435847d74bf12a7b450b70d9326a57c99da6
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 23 16:27:01 2023 +0300

    scripts: add locale update auto
This commit is contained in:
Stanislav Chzhen 2023-04-03 18:22:15 +03:00
parent 2a0d062947
commit 195300f56e
2 changed files with 126 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scripts/README.md
View File

@ -189,6 +189,9 @@ manifest file templates, and helper scripts.
* `go run main.go unused`: show the list of unused strings.
* `go run main.go auto-add`: add locales with additions to the git and
restore locales with deletions.
After the download you'll find the output locales in the `client/src/__locales/`
directory.

150
scripts/translations/main.go
View File

@ -3,6 +3,7 @@
package main
import (
"bufio"
"bytes"
"encoding/json"
"flag"
@ -13,12 +14,14 @@ import (
"net/textproto"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghio"
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/maps"
@ -76,19 +79,19 @@ func main() {
switch os.Args[1] {
case "summary":
err = summary(conf.Languages)
check(err)
case "download":
err = download(uri, projectID, conf.Languages)
check(err)
case "unused":
err = unused()
check(err)
err = unused(conf.LocalizableFiles[0])
case "upload":
err = upload(uri, projectID, conf.BaseLangcode)
check(err)
case "auto-add":
err = autoAdd(conf.LocalizableFiles[0])
default:
usage("unknown command")
}
check(err)
}
// check is a simple error-checking helper for scripts.
@ -108,11 +111,14 @@ Commands:
summary
Print summary.
download [-n <count>]
Download translations. count is a number of concurrent downloads.
Download translations. count is a number of concurrent downloads.
unused
Print unused strings.
upload
Upload translations.`
Upload translations.
auto-add
Add locales with additions to the git and restore locales with
deletions.`
if addStr != "" {
fmt.Printf("%s\n%s\n", addStr, usageStr)
@ -135,6 +141,8 @@ type twoskyConf struct {
// readTwoskyConf returns configuration.
func readTwoskyConf() (t twoskyConf, err error) {
defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }()
b, err := os.ReadFile(twoskyConfFile)
if err != nil {
// Don't wrap the error since it's informative enough as is.
@ -163,6 +171,10 @@ func readTwoskyConf() (t twoskyConf, err error) {
}
}
if len(conf.LocalizableFiles) == 0 {
return twoskyConf{}, errors.Error("no localizable files specified")
}
return conf, nil
}
@ -232,7 +244,7 @@ func download(uri *url.URL, projectID string, langs languages) (err error) {
err = flagSet.Parse(os.Args[2:])
if err != nil {
// Don't wrap the error since there is exit on error.
// Don't wrap the error since it's informative enough as is.
return err
}
@ -246,12 +258,12 @@ func download(uri *url.URL, projectID string, langs languages) (err error) {
Timeout: 10 * time.Second,
}
var wg sync.WaitGroup
wg := &sync.WaitGroup{}
uriCh := make(chan *url.URL, len(langs))
for i := 0; i < numWorker; i++ {
wg.Add(1)
go downloadWorker(&wg, client, uriCh)
go downloadWorker(wg, client, uriCh)
}
for lang := range langs {
@ -342,9 +354,7 @@ func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode)
}
// unused prints unused text labels.
func unused() (err error) {
fileNames := []string{}
basePath := filepath.Join(localesDir, defaultBaseFile)
func unused(basePath string) (err error) {
baseLoc, err := readLocales(basePath)
if err != nil {
return fmt.Errorf("unused: %w", err)
@ -352,9 +362,10 @@ func unused() (err error) {
locDir := filepath.Clean(localesDir)
fileNames := []string{}
err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error {
if err != nil {
log.Info("accessing a path %q: %s", name, err)
log.Info("warning: accessing a path %q: %s", name, err)
return nil
}
@ -379,12 +390,11 @@ func unused() (err error) {
return fmt.Errorf("filepath walking %q: %w", srcDir, err)
}
err = removeUnused(fileNames, baseLoc)
return errors.Annotate(err, "removing unused: %w")
return findUnused(fileNames, baseLoc)
}
func removeUnused(fileNames []string, loc locales) (err error) {
// findUnused prints unused text labels from fileNames.
func findUnused(fileNames []string, loc locales) (err error) {
knownUsed := []textLabel{
"blocking_mode_refused",
"blocking_mode_nxdomain",
@ -399,8 +409,7 @@ func removeUnused(fileNames []string, loc locales) (err error) {
var buf []byte
buf, err = os.ReadFile(fn)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
return fmt.Errorf("finding unused: %w", err)
}
for k := range loc {
@ -410,19 +419,14 @@ func removeUnused(fileNames []string, loc locales) (err error) {
}
}
printUnused(loc)
return nil
}
// printUnused text labels to stdout.
func printUnused(loc locales) {
keys := maps.Keys(loc)
slices.Sort(keys)
for _, v := range keys {
fmt.Println(v)
}
return nil
}
// upload base translation. uri is the base URL. projectID is the name of the
@ -536,3 +540,95 @@ func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
return nil
}
// autoAdd adds locales with additions to the git and restores locales with
// deletions.
func autoAdd(basePath string) (err error) {
defer func() { err = errors.Annotate(err, "auto add: %w") }()
adds, dels, err := changedLocales()
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
if slices.Contains(dels, basePath) {
return errors.Error("base locale contains deletions")
}
var (
args []string
code int
out []byte
)
if len(adds) > 0 {
args = append([]string{"add"}, adds...)
code, out, err = aghos.RunCommand("git", args...)
if err != nil || code != 0 {
return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
}
}
if len(dels) > 0 {
args = append([]string{"restore"}, dels...)
code, out, err = aghos.RunCommand("git", args...)
if err != nil || code != 0 {
return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
}
}
return nil
}
// changedLocales returns cleaned paths of locales with changes or error. adds
// is the list of locales with only additions. dels is the list of locales
// with only deletions.
func changedLocales() (adds, dels []string, err error) {
defer func() { err = errors.Annotate(err, "getting changes: %w") }()
cmd := exec.Command("git", "diff", "--numstat", localesDir)
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, nil, fmt.Errorf("piping: %w", err)
}
err = cmd.Start()
if err != nil {
return nil, nil, fmt.Errorf("starting: %w", err)
}
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
if len(fields) < 3 {
return nil, nil, fmt.Errorf("invalid input: %q", line)
}
path := fields[2]
if fields[0] == "0" {
dels = append(dels, path)
} else if fields[1] == "0" {
adds = append(adds, path)
}
}
err = scanner.Err()
if err != nil {
return nil, nil, fmt.Errorf("scanning: %w", err)
}
err = cmd.Wait()
if err != nil {
return nil, nil, fmt.Errorf("waiting: %w", err)
}
return adds, dels, nil
}