Pull request 2274: AGDNS-2374-slog-scripts
Squashed commit of the following:
commit 257bd542f78d6e06a6b4783a050b573240a2b5ca
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Mon Sep 2 17:33:12 2024 +0300
scripts: slog
This commit is contained in:
Stanislav Chzhen
parent
aab6769fa2
commit
76344f9785
|
|
@ -4,8 +4,10 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
|
|
@ -14,10 +16,13 @@ import (
|
|||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/logutil/slogutil"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
l := slogutil.New(nil)
|
||||
|
||||
urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/services.json"
|
||||
if v, ok := os.LookupEnv("URL"); ok {
|
||||
urlStr = v
|
||||
|
|
@ -33,7 +38,7 @@ func main() {
|
|||
|
||||
resp, err := c.Get(urlStr)
|
||||
check(err)
|
||||
defer log.OnCloserError(resp.Body, log.ERROR)
|
||||
defer slogutil.CloseAndLog(ctx, l, resp.Body, slog.LevelError)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode))
|
||||
|
|
@ -64,7 +69,7 @@ func main() {
|
|||
0o644,
|
||||
)
|
||||
check(err)
|
||||
defer log.OnCloserError(f, log.ERROR)
|
||||
defer slogutil.CloseAndLog(ctx, l, f, slog.LevelError)
|
||||
|
||||
err = tmpl.Execute(f, hlSvcs)
|
||||
check(err)
|
||||
|
|
|
|||
|
|
@ -1,25 +1,26 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/ioutil"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/logutil/slogutil"
|
||||
)
|
||||
|
||||
// download and save all translations.
|
||||
func (c *twoskyClient) download() (err error) {
|
||||
func (c *twoskyClient) download(ctx context.Context, l *slog.Logger) (err error) {
|
||||
var numWorker int
|
||||
|
||||
flagSet := flag.NewFlagSet("download", flag.ExitOnError)
|
||||
|
|
@ -50,7 +51,7 @@ func (c *twoskyClient) download() (err error) {
|
|||
|
||||
for range numWorker {
|
||||
wg.Add(1)
|
||||
go downloadWorker(wg, failed, client, uriCh)
|
||||
go downloadWorker(ctx, l, wg, failed, client, uriCh)
|
||||
}
|
||||
|
||||
for _, lang := range c.langs {
|
||||
|
|
@ -62,13 +63,13 @@ func (c *twoskyClient) download() (err error) {
|
|||
close(uriCh)
|
||||
wg.Wait()
|
||||
|
||||
printFailedLocales(failed)
|
||||
printFailedLocales(ctx, l, failed)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// printFailedLocales prints sorted list of failed downloads, if any.
|
||||
func printFailedLocales(failed *sync.Map) {
|
||||
func printFailedLocales(ctx context.Context, l *slog.Logger, failed *sync.Map) {
|
||||
keys := []string{}
|
||||
failed.Range(func(k, _ any) bool {
|
||||
s, ok := k.(string)
|
||||
|
|
@ -86,12 +87,14 @@ func printFailedLocales(failed *sync.Map) {
|
|||
}
|
||||
|
||||
slices.Sort(keys)
|
||||
log.Info("failed locales: %s", strings.Join(keys, " "))
|
||||
l.InfoContext(ctx, "failed", "locales", keys)
|
||||
}
|
||||
|
||||
// downloadWorker downloads translations by received urls and saves them.
|
||||
// Where failed is a map for storing failed downloads.
|
||||
func downloadWorker(
|
||||
ctx context.Context,
|
||||
l *slog.Logger,
|
||||
wg *sync.WaitGroup,
|
||||
failed *sync.Map,
|
||||
client *http.Client,
|
||||
|
|
@ -103,9 +106,9 @@ func downloadWorker(
|
|||
q := uri.Query()
|
||||
code := q.Get("language")
|
||||
|
||||
err := saveToFile(client, uri, code)
|
||||
err := saveToFile(ctx, l, client, uri, code)
|
||||
if err != nil {
|
||||
log.Error("download: worker: %s", err)
|
||||
l.ErrorContext(ctx, "download worker", slogutil.KeyError, err)
|
||||
failed.Store(code, struct{}{})
|
||||
}
|
||||
}
|
||||
|
|
@ -113,12 +116,16 @@ func downloadWorker(
|
|||
|
||||
// saveToFile downloads translation by url and saves it to a file, or returns
|
||||
// error.
|
||||
func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
|
||||
data, err := getTranslation(client, uri.String())
|
||||
func saveToFile(
|
||||
ctx context.Context,
|
||||
l *slog.Logger,
|
||||
client *http.Client,
|
||||
uri *url.URL,
|
||||
code string,
|
||||
) (err error) {
|
||||
data, err := getTranslation(ctx, l, client, uri.String())
|
||||
if err != nil {
|
||||
log.Info("%s", data)
|
||||
|
||||
return fmt.Errorf("getting translation: %s", err)
|
||||
return fmt.Errorf("getting translation %q: %s", code, err)
|
||||
}
|
||||
|
||||
name := filepath.Join(localesDir, code+".json")
|
||||
|
|
@ -134,13 +141,18 @@ func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
|
|||
|
||||
// getTranslation returns received translation data and error. If err is not
|
||||
// nil, data may contain a response from server for inspection.
|
||||
func getTranslation(client *http.Client, url string) (data []byte, err error) {
|
||||
func getTranslation(
|
||||
ctx context.Context,
|
||||
l *slog.Logger,
|
||||
client *http.Client,
|
||||
url string,
|
||||
) (data []byte, err error) {
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("requesting: %w", err)
|
||||
}
|
||||
|
||||
defer log.OnCloserError(resp.Body, log.ERROR)
|
||||
defer slogutil.CloseAndLog(ctx, l, resp.Body, slog.LevelError)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
|
||||
|
|
|
|||
|
|
@ -6,8 +6,10 @@ import (
|
|||
"bufio"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
|
@ -18,7 +20,7 @@ import (
|
|||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/logutil/slogutil"
|
||||
"golang.org/x/exp/maps"
|
||||
)
|
||||
|
||||
|
|
@ -63,6 +65,9 @@ type textLabel string
|
|||
type locales map[textLabel]string
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
l := slogutil.New(nil)
|
||||
|
||||
if len(os.Args) == 1 {
|
||||
usage("need a command")
|
||||
}
|
||||
|
|
@ -83,9 +88,9 @@ func main() {
|
|||
cli, err = conf.toClient()
|
||||
check(err)
|
||||
|
||||
err = cli.download()
|
||||
err = cli.download(ctx, l)
|
||||
case "unused":
|
||||
err = unused(conf.LocalizableFiles[0])
|
||||
err = unused(ctx, l, conf.LocalizableFiles[0])
|
||||
case "upload":
|
||||
cli, err = conf.toClient()
|
||||
check(err)
|
||||
|
|
@ -322,7 +327,7 @@ func summary(langs languages) (err error) {
|
|||
}
|
||||
|
||||
// unused prints unused text labels.
|
||||
func unused(basePath string) (err error) {
|
||||
func unused(ctx context.Context, l *slog.Logger, basePath string) (err error) {
|
||||
defer func() { err = errors.Annotate(err, "unused: %w") }()
|
||||
|
||||
baseLoc, err := readLocales(basePath)
|
||||
|
|
@ -331,7 +336,7 @@ func unused(basePath string) (err error) {
|
|||
}
|
||||
|
||||
locDir := filepath.Clean(localesDir)
|
||||
js, err := findJS(locDir)
|
||||
js, err := findJS(ctx, l, locDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -340,10 +345,10 @@ func unused(basePath string) (err error) {
|
|||
}
|
||||
|
||||
// findJS returns list of JavaScript and JSON files or error.
|
||||
func findJS(locDir string) (fileNames []string, err error) {
|
||||
func findJS(ctx context.Context, l *slog.Logger, locDir string) (fileNames []string, err error) {
|
||||
walkFn := func(name string, _ os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
log.Info("warning: accessing a path %q: %s", name, err)
|
||||
l.WarnContext(ctx, "accessing a path", slogutil.KeyError, err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue