// translations downloads translations, uploads translations, prints summary // for translations, prints unused strings. package main import ( "bufio" "bytes" "cmp" "encoding/json" "fmt" "net/url" "os" "os/exec" "path/filepath" "slices" "strings" "time" "github.com/AdguardTeam/AdGuardHome/internal/aghos" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" "golang.org/x/exp/maps" ) const ( twoskyConfFile = "./.twosky.json" localesDir = "./client/src/__locales" defaultBaseFile = "en.json" defaultProjectID = "home" srcDir = "./client/src" twoskyURI = "https://twosky.int.agrd.dev/api/v1" readLimit = 1 * 1024 * 1024 uploadTimeout = 20 * time.Second ) // blockerLangCodes is the codes of languages which need to be fully translated. var blockerLangCodes = []langCode{ "de", "en", "es", "fr", "it", "ja", "ko", "pt-br", "pt-pt", "ru", "zh-cn", "zh-tw", } // langCode is a language code. type langCode string // languages is a map, where key is language code and value is display name. type languages map[langCode]string // textlabel is a text label of localization. type textLabel string // locales is a map, where key is text label and value is translation. type locales map[textLabel]string func main() { if len(os.Args) == 1 { usage("need a command") } if os.Args[1] == "help" { usage("") } conf, err := readTwoskyConfig() check(err) var cli *twoskyClient switch os.Args[1] { case "summary": err = summary(conf.Languages) case "download": cli, err = conf.toClient() check(err) err = cli.download() case "unused": err = unused(conf.LocalizableFiles[0]) case "upload": cli, err = conf.toClient() check(err) err = cli.upload() case "auto-add": err = autoAdd(conf.LocalizableFiles[0]) default: usage("unknown command") } check(err) } // check is a simple error-checking helper for scripts. func check(err error) { if err != nil { panic(err) } } // usage prints usage. If addStr is not empty print addStr and exit with code // 1, otherwise exit with code 0. func usage(addStr string) { const usageStr = `Usage: go run main.go [] Commands: help Print usage. summary Print summary. download [-n ] Download translations. count is a number of concurrent downloads. unused Print unused strings. upload Upload translations. auto-add Add locales with additions to the git and restore locales with deletions.` if addStr != "" { fmt.Printf("%s\n%s\n", addStr, usageStr) os.Exit(1) } fmt.Println(usageStr) os.Exit(0) } // twoskyConfig is the configuration structure for localization. type twoskyConfig struct { Languages languages `json:"languages"` ProjectID string `json:"project_id"` BaseLangcode langCode `json:"base_locale"` LocalizableFiles []string `json:"localizable_files"` } // readTwoskyConfig returns twosky configuration. func readTwoskyConfig() (t *twoskyConfig, err error) { defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }() b, err := os.ReadFile(twoskyConfFile) if err != nil { // Don't wrap the error since it's informative enough as is. return nil, err } var tsc []twoskyConfig err = json.Unmarshal(b, &tsc) if err != nil { err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err) return nil, err } if len(tsc) == 0 { err = fmt.Errorf("%q is empty", twoskyConfFile) return nil, err } conf := tsc[0] for _, lang := range conf.Languages { if lang == "" { return nil, errors.Error("language is empty") } } if len(conf.LocalizableFiles) == 0 { return nil, errors.Error("no localizable files specified") } return &conf, nil } // twoskyClient is the twosky client with methods for download and upload // translations. type twoskyClient struct { // uri is the base URL. uri *url.URL // projectID is the name of the project. projectID string // baseLang is the base language code. baseLang langCode // langs is the list of codes of languages to download. langs []langCode } // toClient reads values from environment variables or defaults, validates // them, and returns the twosky client. func (t *twoskyConfig) toClient() (cli *twoskyClient, err error) { defer func() { err = errors.Annotate(err, "filling config: %w") }() uriStr := cmp.Or(os.Getenv("TWOSKY_URI"), twoskyURI) uri, err := url.Parse(uriStr) if err != nil { return nil, err } projectID := cmp.Or(os.Getenv("TWOSKY_PROJECT_ID"), defaultProjectID) baseLang := t.BaseLangcode uLangStr := os.Getenv("UPLOAD_LANGUAGE") if uLangStr != "" { baseLang = langCode(uLangStr) } langs := maps.Keys(t.Languages) dlLangStr := os.Getenv("DOWNLOAD_LANGUAGES") if dlLangStr == "blocker" { langs = blockerLangCodes } else if dlLangStr != "" { var dlLangs []langCode dlLangs, err = validateLanguageStr(dlLangStr, t.Languages) if err != nil { return nil, err } langs = dlLangs } return &twoskyClient{ uri: uri, projectID: projectID, baseLang: baseLang, langs: langs, }, nil } // validateLanguageStr validates languages codes that contain in the str and // returns them or error. func validateLanguageStr(str string, all languages) (langs []langCode, err error) { codes := strings.Fields(str) langs = make([]langCode, 0, len(codes)) for _, k := range codes { lc := langCode(k) _, ok := all[lc] if !ok { return nil, fmt.Errorf("validating languages: unexpected language code %q", k) } langs = append(langs, lc) } return langs, nil } // readLocales reads file with name fn and returns a map, where key is text // label and value is localization. func readLocales(fn string) (loc locales, err error) { b, err := os.ReadFile(fn) if err != nil { // Don't wrap the error since it's informative enough as is. return nil, err } loc = make(locales) err = json.Unmarshal(b, &loc) if err != nil { err = fmt.Errorf("unmarshalling %q: %w", fn, err) return nil, err } return loc, nil } // summary prints summary for translations. func summary(langs languages) (err error) { basePath := filepath.Join(localesDir, defaultBaseFile) baseLoc, err := readLocales(basePath) if err != nil { return fmt.Errorf("summary: %w", err) } size := float64(len(baseLoc)) keys := maps.Keys(langs) slices.Sort(keys) for _, lang := range keys { name := filepath.Join(localesDir, string(lang)+".json") if name == basePath { continue } var loc locales loc, err = readLocales(name) if err != nil { return fmt.Errorf("summary: reading locales: %w", err) } f := float64(len(loc)) * 100 / size blocker := "" // N is small enough to not raise performance questions. ok := slices.Contains(blockerLangCodes, lang) if ok { blocker = " (blocker)" } fmt.Printf("%s\t %6.2f %%%s\n", lang, f, blocker) } return nil } // unused prints unused text labels. func unused(basePath string) (err error) { defer func() { err = errors.Annotate(err, "unused: %w") }() baseLoc, err := readLocales(basePath) if err != nil { return err } locDir := filepath.Clean(localesDir) js, err := findJS(locDir) if err != nil { return err } return findUnused(js, baseLoc) } // findJS returns list of JavaScript and JSON files or error. func findJS(locDir string) (fileNames []string, err error) { walkFn := func(name string, _ os.FileInfo, err error) error { if err != nil { log.Info("warning: accessing a path %q: %s", name, err) return nil } if strings.HasPrefix(name, locDir) { return nil } ext := filepath.Ext(name) if ext == ".js" || ext == ".json" { fileNames = append(fileNames, name) } return nil } err = filepath.Walk(srcDir, walkFn) if err != nil { return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err) } return fileNames, nil } // findUnused prints unused text labels from fileNames. func findUnused(fileNames []string, loc locales) (err error) { knownUsed := []textLabel{ "blocking_mode_refused", "blocking_mode_nxdomain", "blocking_mode_custom_ip", } for _, v := range knownUsed { delete(loc, v) } for _, fn := range fileNames { var buf []byte buf, err = os.ReadFile(fn) if err != nil { return fmt.Errorf("finding unused: %w", err) } for k := range loc { if bytes.Contains(buf, []byte(k)) { delete(loc, k) } } } keys := maps.Keys(loc) slices.Sort(keys) for _, v := range keys { fmt.Println(v) } return nil } // autoAdd adds locales with additions to the git and restores locales with // deletions. func autoAdd(basePath string) (err error) { defer func() { err = errors.Annotate(err, "auto add: %w") }() adds, dels, err := changedLocales() if err != nil { // Don't wrap the error since it's informative enough as is. return err } if slices.Contains(dels, basePath) { return errors.Error("base locale contains deletions") } err = handleAdds(adds) if err != nil { // Don't wrap the error since it's informative enough as is. return nil } err = handleDels(dels) if err != nil { // Don't wrap the error since it's informative enough as is. return nil } return nil } // handleAdds adds locales with additions to the git. func handleAdds(locales []string) (err error) { if len(locales) == 0 { return nil } args := append([]string{"add"}, locales...) code, out, err := aghos.RunCommand("git", args...) if err != nil || code != 0 { return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err) } return nil } // handleDels restores locales with deletions. func handleDels(locales []string) (err error) { if len(locales) == 0 { return nil } args := append([]string{"restore"}, locales...) code, out, err := aghos.RunCommand("git", args...) if err != nil || code != 0 { return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err) } return nil } // changedLocales returns cleaned paths of locales with changes or error. adds // is the list of locales with only additions. dels is the list of locales // with only deletions. func changedLocales() (adds, dels []string, err error) { defer func() { err = errors.Annotate(err, "getting changes: %w") }() cmd := exec.Command("git", "diff", "--numstat", localesDir) stdout, err := cmd.StdoutPipe() if err != nil { return nil, nil, fmt.Errorf("piping: %w", err) } err = cmd.Start() if err != nil { return nil, nil, fmt.Errorf("starting: %w", err) } scanner := bufio.NewScanner(stdout) for scanner.Scan() { line := scanner.Text() fields := strings.Fields(line) if len(fields) < 3 { return nil, nil, fmt.Errorf("invalid input: %q", line) } path := fields[2] if fields[0] == "0" { dels = append(dels, path) } else if fields[1] == "0" { adds = append(adds, path) } } err = scanner.Err() if err != nil { return nil, nil, fmt.Errorf("scanning: %w", err) } err = cmd.Wait() if err != nil { return nil, nil, fmt.Errorf("waiting: %w", err) } return adds, dels, nil }