scripts: add download languages

This commit is contained in:
Stanislav Chzhen 2023-07-05 13:51:40 +03:00
parent c46516475d
commit 55027cfa1c
1 changed files with 70 additions and 5 deletions

View File

@ -254,6 +254,17 @@ func download(uri *url.URL, projectID string, langs languages) (err error) {
usage("count must be positive")
}
langStr := os.Getenv("DOWNLOAD_LANGUAGES")
if langStr != "" {
var dlLangs languages
dlLangs, err = validateLanguageStr(langStr, langs)
if err != nil {
return fmt.Errorf("validating download languages: %w", err)
}
langs = dlLangs
}
downloadURI := uri.JoinPath("download")
client := &http.Client{
@ -261,11 +272,12 @@ func download(uri *url.URL, projectID string, langs languages) (err error) {
}
wg := &sync.WaitGroup{}
failed := &sync.Map{}
uriCh := make(chan *url.URL, len(langs))
for i := 0; i < numWorker; i++ {
wg.Add(1)
go downloadWorker(wg, client, uriCh)
go downloadWorker(wg, failed, client, uriCh)
}
for lang := range langs {
@ -277,25 +289,76 @@ func download(uri *url.URL, projectID string, langs languages) (err error) {
close(uriCh)
wg.Wait()
printFailedLocales(failed)
return nil
}
// validateLanguageStr validates languages codes that contain in the str and
// returns language map, where key is language code and value is display name.
func validateLanguageStr(str string, all languages) (langs languages, err error) {
langs = make(languages)
codes := strings.Fields(str)
for _, k := range codes {
lc := langCode(k)
name, ok := all[lc]
if !ok {
return nil, fmt.Errorf("unexpected language %s", k)
}
langs[lc] = name
}
return langs, nil
}
// printFailedLocales prints sorted list of failed downloads, if any.
func printFailedLocales(failed *sync.Map) {
keys := []string{}
failed.Range(func(k, _ any) bool {
s, ok := k.(string)
if !ok {
panic("unexpected type")
}
keys = append(keys, s)
return true
})
if len(keys) == 0 {
return
}
slices.Sort(keys)
log.Info("failed locales: %s", strings.Join(keys, " "))
}
// downloadWorker downloads translations by received urls and saves them.
func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.URL) {
// Where failed is a map for storing failed downloads.
func downloadWorker(
wg *sync.WaitGroup,
failed *sync.Map,
client *http.Client,
uriCh <-chan *url.URL,
) {
defer wg.Done()
for uri := range uriCh {
q := uri.Query()
code := q.Get("language")
data, err := getTranslation(client, uri.String())
if err != nil {
log.Error("download worker: getting translation: %s", err)
log.Info("download worker: error response:\n%s", data)
failed.Store(code, true)
continue
}
q := uri.Query()
code := q.Get("language")
// Fix some TwoSky weirdnesses.
//
// TODO(a.garipov): Remove when those are fixed.
@ -306,6 +369,8 @@ func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.U
if err != nil {
log.Error("download worker: writing file: %s", err)
failed.Store(code, true)
continue
}