AdGuardHome/scripts/translations/download.go

173 lines
3.6 KiB
Go
Raw Normal View History

package main
import (
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"time"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/ioutil"
"github.com/AdguardTeam/golibs/log"
)
// download and save all translations.
func (c *twoskyClient) download() (err error) {
var numWorker int
flagSet := flag.NewFlagSet("download", flag.ExitOnError)
flagSet.Usage = func() {
usage("download command error")
}
flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
err = flagSet.Parse(os.Args[2:])
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
if numWorker < 1 {
usage("count must be positive")
}
downloadURI := c.uri.JoinPath("download")
client := &http.Client{
Timeout: 10 * time.Second,
}
wg := &sync.WaitGroup{}
failed := &sync.Map{}
uriCh := make(chan *url.URL, len(c.langs))
for i := 0; i < numWorker; i++ {
wg.Add(1)
go downloadWorker(wg, failed, client, uriCh)
}
for _, lang := range c.langs {
uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang)
uriCh <- uri
}
close(uriCh)
wg.Wait()
printFailedLocales(failed)
return nil
}
// printFailedLocales prints sorted list of failed downloads, if any.
func printFailedLocales(failed *sync.Map) {
keys := []string{}
failed.Range(func(k, _ any) bool {
s, ok := k.(string)
if !ok {
panic("unexpected type")
}
keys = append(keys, s)
return true
})
if len(keys) == 0 {
return
}
slices.Sort(keys)
log.Info("failed locales: %s", strings.Join(keys, " "))
}
// downloadWorker downloads translations by received urls and saves them.
// Where failed is a map for storing failed downloads.
func downloadWorker(
wg *sync.WaitGroup,
failed *sync.Map,
client *http.Client,
uriCh <-chan *url.URL,
) {
defer wg.Done()
for uri := range uriCh {
q := uri.Query()
code := q.Get("language")
err := saveToFile(client, uri, code)
if err != nil {
log.Error("download: worker: %s", err)
failed.Store(code, struct{}{})
}
}
}
// saveToFile downloads translation by url and saves it to a file, or returns
// error.
func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
data, err := getTranslation(client, uri.String())
if err != nil {
log.Info("%s", data)
return fmt.Errorf("getting translation: %s", err)
}
name := filepath.Join(localesDir, code+".json")
err = os.WriteFile(name, data, 0o664)
if err != nil {
return fmt.Errorf("writing file: %s", err)
}
fmt.Println(name)
return nil
}
// getTranslation returns received translation data and error. If err is not
// nil, data may contain a response from server for inspection.
func getTranslation(client *http.Client, url string) (data []byte, err error) {
resp, err := client.Get(url)
if err != nil {
return nil, fmt.Errorf("requesting: %w", err)
}
defer log.OnCloserError(resp.Body, log.ERROR)
if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
// Go on and download the body for inspection.
}
limitReader := ioutil.LimitReader(resp.Body, readLimit)
data, readErr := io.ReadAll(limitReader)
return data, errors.WithDeferred(err, readErr)
}
// translationURL returns a new url.URL with provided query parameters.
func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
uri = &url.URL{}
*uri = *oldURL
q := uri.Query()
q.Set("format", "json")
q.Set("filename", baseFile)
q.Set("project", projectID)
q.Set("language", string(lang))
uri.RawQuery = q.Encode()
return uri
}