Pull request 1908: AG-23497-scripts-download-languages

Squashed commit of the following:

commit 874e847fc9bbfaeb8af1c02eb0ba1dbb98bd008f
Merge: 4becdd809 a79deda66
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jul 12 16:01:45 2023 +0300

    Merge branch 'master' into AG-23497-scripts-download-languages

commit 4becdd8092558b15d783674f5b9d1e9c151e3a8c
Merge: 1e5385c33 40884624c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jul 12 13:34:34 2023 +0300

    Merge branch 'master' into AG-23497-scripts-download-languages

commit 1e5385c33a
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Jul 11 19:56:29 2023 +0300

    all: upd golibs, imp code

commit 0498960b00
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jul 7 19:05:58 2023 +0300

    scripts: imp naming

commit 6e36ed83c6
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Jul 6 16:37:13 2023 +0300

    scripts: separate files

commit 55027cfa1c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jul 5 13:51:40 2023 +0300

    scripts: add download languages

go.mod

@@ -5,7 +5,7 @@ go 1.19
 require (
 	// TODO(a.garipov): Update to a tagged version when it's released.
 	github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768
-	github.com/AdguardTeam/golibs v0.13.3
+	github.com/AdguardTeam/golibs v0.13.4
 	github.com/AdguardTeam/urlfilter v0.16.1
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/ameshkov/dnscrypt/v2 v2.2.7

go.sum

@@ -2,8 +2,8 @@ github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768 h1:5Ia6wA+
 github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768/go.mod h1:CQhZTkqC8X0ID6glrtyaxgqRRdiYfn1gJulC1cZ5Dn8=
 github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=
 github.com/AdguardTeam/golibs v0.10.4/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw=
-github.com/AdguardTeam/golibs v0.13.3 h1:RT3QbzThtaLiFLkIUDS6/hlGEXrh0zYvdf4bd7UWpGo=
+github.com/AdguardTeam/golibs v0.13.4 h1:ACTwIR1pEENBijHcEWtiMbSh4wWQOlIHRxmUB8oBHf8=
-github.com/AdguardTeam/golibs v0.13.3/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
+github.com/AdguardTeam/golibs v0.13.4/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
 github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU=
 github.com/AdguardTeam/urlfilter v0.16.1 h1:ZPi0rjqo8cQf2FVdzo6cqumNoHZx2KPXj2yZa1A5BBw=
 github.com/AdguardTeam/urlfilter v0.16.1/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI=

scripts/README.md

@@ -269,25 +269,29 @@ Optional environment:
 
    ###  Usage
 
- *  `go run main.go help`: print usage.
+ *  `go run ./scripts/translations help`: print usage.
 
- *  `go run main.go download [-n <count>]`: download and save all translations.
+ *  `go run ./scripts/translations download [-n <count>]`: download and save
-    `n` is optional flag where count is a number of concurrent downloads.
+    all translations.  `n` is optional flag where count is a number of
+    concurrent downloads.
 
- *  `go run main.go upload`: upload the base `en` locale.
+ *  `go run ./scripts/translations upload`: upload the base `en` locale.
 
- *  `go run main.go summary`: show the current locales summary.
+ *  `go run ./scripts/translations summary`: show the current locales summary.
 
- *  `go run main.go unused`: show the list of unused strings.
+ *  `go run ./scripts/translations unused`: show the list of unused strings.
 
- *  `go run main.go auto-add`: add locales with additions to the git and
+ *  `go run ./scripts/translations auto-add`: add locales with additions to the
-    restore locales with deletions.
+    git and restore locales with deletions.
 
 After the download you'll find the output locales in the `client/src/__locales/`
 directory.
 
 Optional environment:
 
+ *  `DOWNLOAD_LANGUAGES`: set a list of specific languages to `download`.  For
+    example `ar be bg`.
+
  *  `UPLOAD_LANGUAGE`: set an alternative language for `upload`.
 
  *  `TWOSKY_URI`: set an alternative URL for `download` or `upload`.

scripts/make/go-lint.sh

@@ -183,6 +183,7 @@ run_linter gocognit --over 10\
 	./internal/tools/\
 	./internal/version/\
 	./internal/whois/\
+	./scripts/\
 	;
 
 run_linter ineffassign ./...

scripts/translations/download.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/aghio"
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/log"
+	"golang.org/x/exp/slices"
+)
+
+// download and save all translations.
+func (c *twoskyClient) download() (err error) {
+	var numWorker int
+
+	flagSet := flag.NewFlagSet("download", flag.ExitOnError)
+	flagSet.Usage = func() {
+		usage("download command error")
+	}
+	flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
+
+	err = flagSet.Parse(os.Args[2:])
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return err
+	}
+
+	if numWorker < 1 {
+		usage("count must be positive")
+	}
+
+	downloadURI := c.uri.JoinPath("download")
+
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	wg := &sync.WaitGroup{}
+	failed := &sync.Map{}
+	uriCh := make(chan *url.URL, len(c.langs))
+
+	for i := 0; i < numWorker; i++ {
+		wg.Add(1)
+		go downloadWorker(wg, failed, client, uriCh)
+	}
+
+	for lang := range c.langs {
+		uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang)
+
+		uriCh <- uri
+	}
+
+	close(uriCh)
+	wg.Wait()
+
+	printFailedLocales(failed)
+
+	return nil
+}
+
+// printFailedLocales prints sorted list of failed downloads, if any.
+func printFailedLocales(failed *sync.Map) {
+	keys := []string{}
+	failed.Range(func(k, _ any) bool {
+		s, ok := k.(string)
+		if !ok {
+			panic("unexpected type")
+		}
+
+		keys = append(keys, s)
+
+		return true
+	})
+
+	if len(keys) == 0 {
+		return
+	}
+
+	slices.Sort(keys)
+	log.Info("failed locales: %s", strings.Join(keys, " "))
+}
+
+// downloadWorker downloads translations by received urls and saves them.
+// Where failed is a map for storing failed downloads.
+func downloadWorker(
+	wg *sync.WaitGroup,
+	failed *sync.Map,
+	client *http.Client,
+	uriCh <-chan *url.URL,
+) {
+	defer wg.Done()
+
+	for uri := range uriCh {
+		q := uri.Query()
+		code := q.Get("language")
+
+		err := saveToFile(client, uri, code)
+		if err != nil {
+			log.Error("download: worker: %s", err)
+			failed.Store(code, struct{}{})
+		}
+	}
+}
+
+// saveToFile downloads translation by url and saves it to a file, or returns
+// error.
+func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
+	data, err := getTranslation(client, uri.String())
+	if err != nil {
+		log.Info("%s", data)
+
+		return fmt.Errorf("getting translation: %s", err)
+	}
+
+	name := filepath.Join(localesDir, code+".json")
+	err = os.WriteFile(name, data, 0o664)
+	if err != nil {
+		return fmt.Errorf("writing file: %s", err)
+	}
+
+	fmt.Println(name)
+
+	return nil
+}
+
+// getTranslation returns received translation data and error.  If err is not
+// nil, data may contain a response from server for inspection.
+func getTranslation(client *http.Client, url string) (data []byte, err error) {
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("requesting: %w", err)
+	}
+
+	defer log.OnCloserError(resp.Body, log.ERROR)
+
+	if resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
+
+		// Go on and download the body for inspection.
+	}
+
+	limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
+	if lrErr != nil {
+		// Generally shouldn't happen, since the only error returned by
+		// [aghio.LimitReader] is an argument error.
+		panic(fmt.Errorf("limit reading: %w", lrErr))
+	}
+
+	data, readErr := io.ReadAll(limitReader)
+
+	return data, errors.WithDeferred(err, readErr)
+}
+
+// translationURL returns a new url.URL with provided query parameters.
+func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
+	uri = &url.URL{}
+	*uri = *oldURL
+
+	q := uri.Query()
+	q.Set("format", "json")
+	q.Set("filename", baseFile)
+	q.Set("project", projectID)
+	q.Set("language", string(lang))
+
+	uri.RawQuery = q.Encode()
+
+	return uri
+}

scripts/translations/main.go

@@ -6,25 +6,16 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
-	"flag"
 	"fmt"
-	"io"
-	"mime/multipart"
-	"net/http"
-	"net/textproto"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
-	"sync"
 	"time"
 
-	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
-	"github.com/AdguardTeam/AdGuardHome/internal/aghio"
 	"github.com/AdguardTeam/AdGuardHome/internal/aghos"
 	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/httphdr"
 	"github.com/AdguardTeam/golibs/log"
 	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
@@ -39,6 +30,7 @@ const (
 	twoskyURI        = "https://twosky.int.agrd.dev/api/v1"
 
 	readLimit     = 1 * 1024 * 1024
+	uploadTimeout = 10 * time.Second
 )
 
 // langCode is a language code.
@@ -62,31 +54,26 @@ func main() {
 		usage("")
 	}
 
-	uriStr := os.Getenv("TWOSKY_URI")
+	conf, err := readTwoskyConfig()
-	if uriStr == "" {
-		uriStr = twoskyURI
-	}
-
-	uri, err := url.Parse(uriStr)
 	check(err)
 
-	projectID := os.Getenv("TWOSKY_PROJECT_ID")
+	var cli *twoskyClient
-	if projectID == "" {
-		projectID = defaultProjectID
-	}
-
-	conf, err := readTwoskyConf()
-	check(err)
 
 	switch os.Args[1] {
 	case "summary":
 		err = summary(conf.Languages)
 	case "download":
-		err = download(uri, projectID, conf.Languages)
+		cli, err = conf.toClient()
+		check(err)
+
+		err = cli.download()
 	case "unused":
 		err = unused(conf.LocalizableFiles[0])
 	case "upload":
-		err = upload(uri, projectID, conf.BaseLangcode)
+		cli, err = conf.toClient()
+		check(err)
+
+		err = cli.upload()
 	case "auto-add":
 		err = autoAdd(conf.LocalizableFiles[0])
 	default:
@@ -133,51 +120,131 @@ Commands:
 	os.Exit(0)
 }
 
-// twoskyConf is the configuration structure for localization.
+// twoskyConfig is the configuration structure for localization.
-type twoskyConf struct {
+type twoskyConfig struct {
 	Languages        languages `json:"languages"`
 	ProjectID        string    `json:"project_id"`
 	BaseLangcode     langCode  `json:"base_locale"`
 	LocalizableFiles []string  `json:"localizable_files"`
 }
 
-// readTwoskyConf returns configuration.
+// readTwoskyConfig returns twosky configuration.
-func readTwoskyConf() (t twoskyConf, err error) {
+func readTwoskyConfig() (t *twoskyConfig, err error) {
-	defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }()
+	defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }()
 
 	b, err := os.ReadFile(twoskyConfFile)
 	if err != nil {
 		// Don't wrap the error since it's informative enough as is.
-		return twoskyConf{}, err
+		return nil, err
 	}
 
-	var tsc []twoskyConf
+	var tsc []twoskyConfig
 	err = json.Unmarshal(b, &tsc)
 	if err != nil {
 		err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
 
-		return twoskyConf{}, err
+		return nil, err
 	}
 
 	if len(tsc) == 0 {
 		err = fmt.Errorf("%q is empty", twoskyConfFile)
 
-		return twoskyConf{}, err
+		return nil, err
 	}
 
 	conf := tsc[0]
 
 	for _, lang := range conf.Languages {
 		if lang == "" {
-			return twoskyConf{}, errors.Error("language is empty")
+			return nil, errors.Error("language is empty")
 		}
 	}
 
 	if len(conf.LocalizableFiles) == 0 {
-		return twoskyConf{}, errors.Error("no localizable files specified")
+		return nil, errors.Error("no localizable files specified")
 	}
 
-	return conf, nil
+	return &conf, nil
+}
+
+// twoskyClient is the twosky client with methods for download and upload
+// translations.
+type twoskyClient struct {
+	// uri is the base URL.
+	uri *url.URL
+
+	// langs is the map of languages to download.
+	langs languages
+
+	// projectID is the name of the project.
+	projectID string
+
+	// baseLang is the base language code.
+	baseLang langCode
+}
+
+// toClient reads values from environment variables or defaults, validates
+// them, and returns the twosky client.
+func (t *twoskyConfig) toClient() (cli *twoskyClient, err error) {
+	defer func() { err = errors.Annotate(err, "filling config: %w") }()
+
+	uriStr := os.Getenv("TWOSKY_URI")
+	if uriStr == "" {
+		uriStr = twoskyURI
+	}
+	uri, err := url.Parse(uriStr)
+	if err != nil {
+		return nil, err
+	}
+
+	projectID := os.Getenv("TWOSKY_PROJECT_ID")
+	if projectID == "" {
+		projectID = defaultProjectID
+	}
+
+	baseLang := t.BaseLangcode
+	uLangStr := os.Getenv("UPLOAD_LANGUAGE")
+	if uLangStr != "" {
+		baseLang = langCode(uLangStr)
+	}
+
+	langs := t.Languages
+	dlLangStr := os.Getenv("DOWNLOAD_LANGUAGES")
+	if dlLangStr != "" {
+		var dlLangs languages
+		dlLangs, err = validateLanguageStr(dlLangStr, langs)
+		if err != nil {
+			return nil, err
+		}
+
+		langs = dlLangs
+	}
+
+	return &twoskyClient{
+		uri:       uri,
+		projectID: projectID,
+		baseLang:  baseLang,
+		langs:     langs,
+	}, nil
+}
+
+// validateLanguageStr validates languages codes that contain in the str and
+// returns language map, where key is language code and value is display name.
+func validateLanguageStr(str string, all languages) (langs languages, err error) {
+	langs = make(languages)
+	codes := strings.Fields(str)
+
+	for _, k := range codes {
+		lc := langCode(k)
+		name, ok := all[lc]
+		if !ok {
+			return nil, fmt.Errorf("validating languages: unexpected language code %q", k)
+		}
+
+		langs[lc] = name
+	}
+
+	return langs, nil
 }
 
 // readLocales reads file with name fn and returns a map, where key is text
@@ -233,163 +300,33 @@ func summary(langs languages) (err error) {
 	return nil
 }
 
-// download and save all translations.  uri is the base URL.  projectID is the
+// unused prints unused text labels.
-// name of the project.
+func unused(basePath string) (err error) {
-func download(uri *url.URL, projectID string, langs languages) (err error) {
+	defer func() { err = errors.Annotate(err, "unused: %w") }()
-	var numWorker int
 
-	flagSet := flag.NewFlagSet("download", flag.ExitOnError)
+	baseLoc, err := readLocales(basePath)
-	flagSet.Usage = func() {
-		usage("download command error")
-	}
-	flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
-
-	err = flagSet.Parse(os.Args[2:])
 	if err != nil {
-		// Don't wrap the error since it's informative enough as is.
 		return err
 	}
 
-	if numWorker < 1 {
-		usage("count must be positive")
-	}
-
-	downloadURI := uri.JoinPath("download")
-
-	client := &http.Client{
-		Timeout: 10 * time.Second,
-	}
-
-	wg := &sync.WaitGroup{}
-	uriCh := make(chan *url.URL, len(langs))
-
-	for i := 0; i < numWorker; i++ {
-		wg.Add(1)
-		go downloadWorker(wg, client, uriCh)
-	}
-
-	for lang := range langs {
-		uri = translationURL(downloadURI, defaultBaseFile, projectID, lang)
-
-		uriCh <- uri
-	}
-
-	close(uriCh)
-	wg.Wait()
-
-	return nil
-}
-
-// downloadWorker downloads translations by received urls and saves them.
-func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.URL) {
-	defer wg.Done()
-
-	for uri := range uriCh {
-		data, err := getTranslation(client, uri.String())
-		if err != nil {
-			log.Error("download worker: getting translation: %s", err)
-			log.Info("download worker: error response:\n%s", data)
-
-			continue
-		}
-
-		q := uri.Query()
-		code := q.Get("language")
-
-		// Fix some TwoSky weirdnesses.
-		//
-		// TODO(a.garipov): Remove when those are fixed.
-		code = strings.ToLower(code)
-
-		name := filepath.Join(localesDir, code+".json")
-		err = os.WriteFile(name, data, 0o664)
-		if err != nil {
-			log.Error("download worker: writing file: %s", err)
-
-			continue
-		}
-
-		fmt.Println(name)
-	}
-}
-
-// getTranslation returns received translation data and error.  If err is not
-// nil, data may contain a response from server for inspection.
-func getTranslation(client *http.Client, url string) (data []byte, err error) {
-	resp, err := client.Get(url)
-	if err != nil {
-		return nil, fmt.Errorf("requesting: %w", err)
-	}
-
-	defer log.OnCloserError(resp.Body, log.ERROR)
-
-	if resp.StatusCode != http.StatusOK {
-		err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
-
-		// Go on and download the body for inspection.
-	}
-
-	limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
-	if lrErr != nil {
-		// Generally shouldn't happen, since the only error returned by
-		// [aghio.LimitReader] is an argument error.
-		panic(fmt.Errorf("limit reading: %w", lrErr))
-	}
-
-	data, readErr := io.ReadAll(limitReader)
-
-	return data, errors.WithDeferred(err, readErr)
-}
-
-// translationURL returns a new url.URL with provided query parameters.
-func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
-	uri = &url.URL{}
-	*uri = *oldURL
-
-	// Fix some TwoSky weirdnesses.
-	//
-	// TODO(a.garipov): Remove when those are fixed.
-	switch lang {
-	case "si-lk":
-		lang = "si-LK"
-	case "zh-hk":
-		lang = "zh-HK"
-	default:
-		// Go on.
-	}
-
-	q := uri.Query()
-	q.Set("format", "json")
-	q.Set("filename", baseFile)
-	q.Set("project", projectID)
-	q.Set("language", string(lang))
-
-	uri.RawQuery = q.Encode()
-
-	return uri
-}
-
-// unused prints unused text labels.
-func unused(basePath string) (err error) {
-	baseLoc, err := readLocales(basePath)
-	if err != nil {
-		return fmt.Errorf("unused: %w", err)
-	}
-
 	locDir := filepath.Clean(localesDir)
+	js, err := findJS(locDir)
+	if err != nil {
+		return err
+	}
 
-	fileNames := []string{}
+	return findUnused(js, baseLoc)
-	err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error {
+}
+
+// findJS returns list of JavaScript and JSON files or error.
+func findJS(locDir string) (fileNames []string, err error) {
+	walkFn := func(name string, _ os.FileInfo, err error) error {
 		if err != nil {
 			log.Info("warning: accessing a path %q: %s", name, err)
 
 			return nil
 		}
 
-		if info.IsDir() {
-			return nil
-		}
-
 		if strings.HasPrefix(name, locDir) {
 			return nil
 		}
@@ -400,13 +337,14 @@ func unused(basePath string) (err error) {
 		}
 
 		return nil
-	})
-
-	if err != nil {
-		return fmt.Errorf("filepath walking %q: %w", srcDir, err)
 	}
 
-	return findUnused(fileNames, baseLoc)
+	err = filepath.Walk(srcDir, walkFn)
+	if err != nil {
+		return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err)
+	}
+
+	return fileNames, nil
 }
 
 // findUnused prints unused text labels from fileNames.
@@ -445,118 +383,6 @@ func findUnused(fileNames []string, loc locales) (err error) {
 	return nil
 }
 
-// upload base translation.  uri is the base URL.  projectID is the name of the
-// project.  baseLang is the base language code.
-func upload(uri *url.URL, projectID string, baseLang langCode) (err error) {
-	defer func() { err = errors.Annotate(err, "upload: %w") }()
-
-	uploadURI := uri.JoinPath("upload")
-
-	lang := baseLang
-
-	langStr := os.Getenv("UPLOAD_LANGUAGE")
-	if langStr != "" {
-		lang = langCode(langStr)
-	}
-
-	basePath := filepath.Join(localesDir, defaultBaseFile)
-
-	formData := map[string]string{
-		"format":   "json",
-		"language": string(lang),
-		"filename": defaultBaseFile,
-		"project":  projectID,
-	}
-
-	buf, cType, err := prepareMultipartMsg(formData, basePath)
-	if err != nil {
-		return fmt.Errorf("preparing multipart msg: %w", err)
-	}
-
-	err = send(uploadURI.String(), cType, buf)
-	if err != nil {
-		return fmt.Errorf("sending multipart msg: %w", err)
-	}
-
-	return nil
-}
-
-// prepareMultipartMsg prepares translation data for upload.
-func prepareMultipartMsg(
-	formData map[string]string,
-	basePath string,
-) (buf *bytes.Buffer, cType string, err error) {
-	buf = &bytes.Buffer{}
-	w := multipart.NewWriter(buf)
-	var fw io.Writer
-
-	for k, v := range formData {
-		err = w.WriteField(k, v)
-		if err != nil {
-			return nil, "", fmt.Errorf("writing field: %w", err)
-		}
-	}
-
-	file, err := os.Open(basePath)
-	if err != nil {
-		return nil, "", fmt.Errorf("opening file: %w", err)
-	}
-
-	defer func() {
-		err = errors.WithDeferred(err, file.Close())
-	}()
-
-	h := make(textproto.MIMEHeader)
-	h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
-
-	d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
-	h.Set(httphdr.ContentDisposition, d)
-
-	fw, err = w.CreatePart(h)
-	if err != nil {
-		return nil, "", fmt.Errorf("creating part: %w", err)
-	}
-
-	_, err = io.Copy(fw, file)
-	if err != nil {
-		return nil, "", fmt.Errorf("copying: %w", err)
-	}
-
-	err = w.Close()
-	if err != nil {
-		return nil, "", fmt.Errorf("closing writer: %w", err)
-	}
-
-	return buf, w.FormDataContentType(), nil
-}
-
-// send POST request to uriStr.
-func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
-	var client http.Client
-
-	req, err := http.NewRequest(http.MethodPost, uriStr, buf)
-	if err != nil {
-		return fmt.Errorf("bad request: %w", err)
-	}
-
-	req.Header.Set(httphdr.ContentType, cType)
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("client post form: %w", err)
-	}
-
-	defer func() {
-		err = errors.WithDeferred(err, resp.Body.Close())
-	}()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
-	}
-
-	return nil
-}
-
 // autoAdd adds locales with additions to the git and restores locales with
 // deletions.
 func autoAdd(basePath string) (err error) {
@@ -572,29 +398,49 @@ func autoAdd(basePath string) (err error) {
 		return errors.Error("base locale contains deletions")
 	}
 
-	var (
+	err = handleAdds(adds)
-		args []string
+	if err != nil {
-		code int
+		// Don't wrap the error since it's informative enough as is.
-		out  []byte
+		return nil
-	)
+	}
 
-	if len(adds) > 0 {
+	err = handleDels(dels)
-		args = append([]string{"add"}, adds...)
+	if err != nil {
-		code, out, err = aghos.RunCommand("git", args...)
+		// Don't wrap the error since it's informative enough as is.
+		return nil
+	}
+
+	return nil
+}
+
+// handleAdds adds locales with additions to the git.
+func handleAdds(locales []string) (err error) {
+	if len(locales) == 0 {
+		return nil
+	}
+
+	args := append([]string{"add"}, locales...)
+	code, out, err := aghos.RunCommand("git", args...)
 
 	if err != nil || code != 0 {
 		return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
 	}
+
+	return nil
+}
+
+// handleDels restores locales with deletions.
+func handleDels(locales []string) (err error) {
+	if len(locales) == 0 {
+		return nil
 	}
 
-	if len(dels) > 0 {
+	args := append([]string{"restore"}, locales...)
-		args = append([]string{"restore"}, dels...)
+	code, out, err := aghos.RunCommand("git", args...)
-		code, out, err = aghos.RunCommand("git", args...)
 
 	if err != nil || code != 0 {
 		return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
 	}
-	}
 
 	return nil
 }

scripts/translations/upload.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"os"
+	"path/filepath"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/httphdr"
+	"github.com/AdguardTeam/golibs/mapsutil"
+)
+
+// upload base translation.
+func (c *twoskyClient) upload() (err error) {
+	defer func() { err = errors.Annotate(err, "upload: %w") }()
+
+	uploadURI := c.uri.JoinPath("upload")
+	basePath := filepath.Join(localesDir, defaultBaseFile)
+
+	formData := map[string]string{
+		"format":   "json",
+		"language": string(c.baseLang),
+		"filename": defaultBaseFile,
+		"project":  c.projectID,
+	}
+
+	buf, cType, err := prepareMultipartMsg(formData, basePath)
+	if err != nil {
+		return fmt.Errorf("preparing multipart msg: %w", err)
+	}
+
+	err = send(uploadURI.String(), cType, buf)
+	if err != nil {
+		return fmt.Errorf("sending multipart msg: %w", err)
+	}
+
+	return nil
+}
+
+// prepareMultipartMsg prepares translation data for upload.
+func prepareMultipartMsg(
+	formData map[string]string,
+	basePath string,
+) (buf *bytes.Buffer, cType string, err error) {
+	buf = &bytes.Buffer{}
+	w := multipart.NewWriter(buf)
+	var fw io.Writer
+
+	err = mapsutil.OrderedRangeError(formData, w.WriteField)
+	if err != nil {
+		return nil, "", fmt.Errorf("writing field: %w", err)
+	}
+
+	file, err := os.Open(basePath)
+	if err != nil {
+		return nil, "", fmt.Errorf("opening file: %w", err)
+	}
+
+	defer func() {
+		err = errors.WithDeferred(err, file.Close())
+	}()
+
+	h := make(textproto.MIMEHeader)
+	h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
+
+	d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
+	h.Set(httphdr.ContentDisposition, d)
+
+	fw, err = w.CreatePart(h)
+	if err != nil {
+		return nil, "", fmt.Errorf("creating part: %w", err)
+	}
+
+	_, err = io.Copy(fw, file)
+	if err != nil {
+		return nil, "", fmt.Errorf("copying: %w", err)
+	}
+
+	err = w.Close()
+	if err != nil {
+		return nil, "", fmt.Errorf("closing writer: %w", err)
+	}
+
+	return buf, w.FormDataContentType(), nil
+}
+
+// send POST request to uriStr.
+func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
+	client := http.Client{
+		Timeout: uploadTimeout,
+	}
+
+	req, err := http.NewRequest(http.MethodPost, uriStr, buf)
+	if err != nil {
+		return fmt.Errorf("bad request: %w", err)
+	}
+
+	req.Header.Set(httphdr.ContentType, cType)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("client post form: %w", err)
+	}
+
+	defer func() {
+		err = errors.WithDeferred(err, resp.Body.Close())
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
+	}
+
+	return nil
+}