scripts: imp naming

This commit is contained in:
Stanislav Chzhen 2023-07-07 19:05:58 +03:00
parent 6e36ed83c6
commit 0498960b00
3 changed files with 44 additions and 47 deletions

View File

@ -18,9 +18,8 @@ import (
"golang.org/x/exp/slices"
)
// download and save all translations. uri is the base URL. projectID is the
// name of the project.
func download(conf *config) (err error) {
// download and save all translations.
func (c *twoskyClient) download() (err error) {
var numWorker int
flagSet := flag.NewFlagSet("download", flag.ExitOnError)
@ -39,7 +38,7 @@ func download(conf *config) (err error) {
usage("count must be positive")
}
downloadURI := conf.uri.JoinPath("download")
downloadURI := c.uri.JoinPath("download")
client := &http.Client{
Timeout: 10 * time.Second,
@ -47,15 +46,15 @@ func download(conf *config) (err error) {
wg := &sync.WaitGroup{}
failed := &sync.Map{}
uriCh := make(chan *url.URL, len(conf.langs))
uriCh := make(chan *url.URL, len(c.langs))
for i := 0; i < numWorker; i++ {
wg.Add(1)
go downloadWorker(wg, failed, client, uriCh)
}
for lang := range conf.langs {
uri := translationURL(downloadURI, defaultBaseFile, conf.projectID, lang)
for lang := range c.langs {
uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang)
uriCh <- uri
}
@ -104,7 +103,7 @@ func downloadWorker(
q := uri.Query()
code := q.Get("language")
err := getTranslationAndSave(client, uri, code)
err := saveToFile(client, uri, code)
if err != nil {
log.Error("download: worker: %s", err)
failed.Store(code, struct{}{})
@ -112,18 +111,15 @@ func downloadWorker(
}
}
// getTranslationAndSave downloads translation by url and saves it, or returns
// saveToFile downloads translation by url and saves it to a file, or returns
// error.
func getTranslationAndSave(client *http.Client, uri *url.URL, code string) (err error) {
func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
data, err := getTranslation(client, uri.String())
if err != nil {
return fmt.Errorf("getting translation: response: %s: %s", data, err)
}
log.Info("%s", data)
// Fix some TwoSky weirdnesses.
//
// TODO(a.garipov): Remove when those are fixed.
code = strings.ToLower(code)
return fmt.Errorf("getting translation: %s", err)
}
name := filepath.Join(localesDir, code+".json")
err = os.WriteFile(name, data, 0o664)
@ -131,6 +127,8 @@ func getTranslationAndSave(client *http.Client, uri *url.URL, code string) (err
return fmt.Errorf("writing file: %s", err)
}
fmt.Println(name)
return nil
}
@ -167,18 +165,6 @@ func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode)
uri = &url.URL{}
*uri = *oldURL
// Fix some TwoSky weirdnesses.
//
// TODO(a.garipov): Remove when those are fixed.
switch lang {
case "si-lk":
lang = "si-LK"
case "zh-hk":
lang = "zh-HK"
default:
// Go on.
}
q := uri.Query()
q.Set("format", "json")
q.Set("filename", baseFile)

View File

@ -12,6 +12,7 @@ import (
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/golibs/errors"
@ -28,7 +29,8 @@ const (
srcDir = "./client/src"
twoskyURI = "https://twosky.int.agrd.dev/api/v1"
readLimit = 1 * 1024 * 1024
readLimit = 1 * 1024 * 1024
uploadTimeout = 10 * time.Second
)
// langCode is a language code.
@ -55,18 +57,23 @@ func main() {
twosky, err := readTwoskyConf()
check(err)
conf, err := twosky.toInternal()
check(err)
var client *twoskyClient
switch os.Args[1] {
case "summary":
err = summary(twosky.Languages)
case "download":
err = download(conf)
client, err = twosky.toClient()
check(err)
err = client.download()
case "unused":
err = unused(twosky.LocalizableFiles[0])
case "upload":
err = upload(conf)
client, err = twosky.toClient()
check(err)
err = client.upload()
case "auto-add":
err = autoAdd(twosky.LocalizableFiles[0])
default:
@ -160,24 +167,25 @@ func readTwoskyConf() (t *twoskyConf, err error) {
return &conf, nil
}
// config is the internal configuration structure.
type config struct {
// twoskyClient is the twosky client with methods for download and upload
// translations.
type twoskyClient struct {
// uri is the base URL.
uri *url.URL
// langs is the map of languages to download.
langs languages
// projectID is the name of the project.
projectID string
// baseLang is the base language code.
baseLang langCode
// langs is the map of languages to download.
langs languages
}
// toInternal reads values from environment variables or defaults, validates
// them, and returns the config.
func (t *twoskyConf) toInternal() (conf *config, err error) {
// toClient reads values from environment variables or defaults, validates
// them, and returns the twosky client.
func (t *twoskyConf) toClient() (conf *twoskyClient, err error) {
defer func() { err = errors.Annotate(err, "filling config: %w") }()
uriStr := os.Getenv("TWOSKY_URI")
@ -212,7 +220,7 @@ func (t *twoskyConf) toInternal() (conf *config, err error) {
langs = dlLangs
}
return &config{
return &twoskyClient{
uri: uri,
projectID: projectID,
baseLang: baseLang,

View File

@ -16,17 +16,17 @@ import (
)
// upload base translation.
func upload(conf *config) (err error) {
func (c *twoskyClient) upload() (err error) {
defer func() { err = errors.Annotate(err, "upload: %w") }()
uploadURI := conf.uri.JoinPath("upload")
uploadURI := c.uri.JoinPath("upload")
basePath := filepath.Join(localesDir, defaultBaseFile)
formData := map[string]string{
"format": "json",
"language": string(conf.baseLang),
"language": string(c.baseLang),
"filename": defaultBaseFile,
"project": conf.projectID,
"project": c.projectID,
}
buf, cType, err := prepareMultipartMsg(formData, basePath)
@ -51,6 +51,7 @@ func prepareMultipartMsg(
w := multipart.NewWriter(buf)
var fw io.Writer
// TODO(s.chzhen): Consider using OrderedRangeError from [agdmaps] package.
for k, v := range formData {
err = w.WriteField(k, v)
if err != nil {
@ -93,7 +94,9 @@ func prepareMultipartMsg(
// send POST request to uriStr.
func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
var client http.Client
client := http.Client{
Timeout: uploadTimeout,
}
req, err := http.NewRequest(http.MethodPost, uriStr, buf)
if err != nil {