diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b060ada..61f8706a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ NOTE: Add new changes BELOW THIS COMMENT. ### Added +- The ability to set custom IP for EDNS Client Subnet by using the DNS-server + configuration section on the DNS settings page in the UI ([#1472]). - The ability to manage safesearch for each service by using the new `safe_search` field ([#1163]). @@ -68,6 +70,7 @@ In this release, the schema version has changed from 17 to 19. ([#5584]). [#1163]: https://github.com/AdguardTeam/AdGuardHome/issues/1163 +[#1472]: https://github.com/AdguardTeam/AdGuardHome/issues/1472 [#5567]: https://github.com/AdguardTeam/AdGuardHome/issues/5567 [#5584]: https://github.com/AdguardTeam/AdGuardHome/issues/5584 diff --git a/client/src/__locales/en.json b/client/src/__locales/en.json index 2afa38bd..5ccd771b 100644 --- a/client/src/__locales/en.json +++ b/client/src/__locales/en.json @@ -290,6 +290,8 @@ "rate_limit": "Rate limit", "edns_enable": "Enable EDNS client subnet", "edns_cs_desc": "Add the EDNS Client Subnet option (ECS) to upstream requests and log the values sent by the clients in the query log.", + "edns_use_custom_ip": "Use custom IP for EDNS", + "edns_use_custom_ip_desc": "Allow to use custom IP for EDNS", "rate_limit_desc": "The number of requests per second allowed per client. Setting it to 0 means no limit.", "blocking_ipv4_desc": "IP address to be returned for a blocked A request", "blocking_ipv6_desc": "IP address to be returned for a blocked AAAA request", @@ -642,5 +644,6 @@ "anonymizer_notification": "<0>Note: IP anonymization is enabled. You can disable it in <1>General settings.", "confirm_dns_cache_clear": "Are you sure you want to clear DNS cache?", "cache_cleared": "DNS cache successfully cleared", - "clear_cache": "Clear cache" + "clear_cache": "Clear cache", + "make_static": "Make static" } diff --git a/client/src/components/Dashboard/BlockedDomains.js b/client/src/components/Dashboard/BlockedDomains.js index 144f5bee..73829a1b 100644 --- a/client/src/components/Dashboard/BlockedDomains.js +++ b/client/src/components/Dashboard/BlockedDomains.js @@ -29,8 +29,11 @@ const BlockedDomains = ({ blockedFiltering, replacedSafebrowsing, replacedParental, + replacedSafesearch, }) => { - const totalBlocked = blockedFiltering + replacedSafebrowsing + replacedParental; + const totalBlocked = ( + blockedFiltering + replacedSafebrowsing + replacedParental + replacedSafesearch + ); return ( diff --git a/client/src/components/Settings/Dhcp/Leases.js b/client/src/components/Settings/Dhcp/Leases.js index 70400538..96ca8852 100644 --- a/client/src/components/Settings/Dhcp/Leases.js +++ b/client/src/components/Settings/Dhcp/Leases.js @@ -1,9 +1,11 @@ import React, { Component } from 'react'; +import { connect } from 'react-redux'; import PropTypes from 'prop-types'; import ReactTable from 'react-table'; import { Trans, withTranslation } from 'react-i18next'; import { LEASES_TABLE_DEFAULT_PAGE_SIZE } from '../../../helpers/constants'; import { sortIp } from '../../../helpers/helpers'; +import { toggleLeaseModal } from '../../../actions'; class Leases extends Component { cellWrap = ({ value }) => ( @@ -14,6 +16,30 @@ class Leases extends Component { ); + convertToStatic = (data) => () => { + const { dispatch } = this.props; + dispatch(toggleLeaseModal(data)); + } + + makeStatic = ({ row }) => { + const { t, disabledLeasesButton } = this.props; + return ( +
+ +
+ ); + } + render() { const { leases, t } = this.props; return ( @@ -23,20 +49,27 @@ class Leases extends Component { { Header: 'MAC', accessor: 'mac', + minWidth: 180, Cell: this.cellWrap, }, { Header: 'IP', accessor: 'ip', + minWidth: 230, Cell: this.cellWrap, sortMethod: sortIp, }, { Header: dhcp_table_hostname, accessor: 'hostname', + minWidth: 230, Cell: this.cellWrap, }, { Header: dhcp_table_expires, accessor: 'expires', + minWidth: 220, Cell: this.cellWrap, + }, { + Header: actions_table_header, + Cell: this.makeStatic, }, ]} pageSize={LEASES_TABLE_DEFAULT_PAGE_SIZE} @@ -53,6 +86,8 @@ class Leases extends Component { Leases.propTypes = { leases: PropTypes.array, t: PropTypes.func, + dispatch: PropTypes.func, + disabledLeasesButton: PropTypes.bool, }; -export default withTranslation()(Leases); +export default withTranslation()(connect(() => ({}), (dispatch) => ({ dispatch }))(Leases)); diff --git a/client/src/components/Settings/Dhcp/StaticLeases/Form.js b/client/src/components/Settings/Dhcp/StaticLeases/Form.js index 0525f6a3..e26b4da5 100644 --- a/client/src/components/Settings/Dhcp/StaticLeases/Form.js +++ b/client/src/components/Settings/Dhcp/StaticLeases/Form.js @@ -2,7 +2,7 @@ import React from 'react'; import PropTypes from 'prop-types'; import { Field, reduxForm } from 'redux-form'; import { Trans, useTranslation } from 'react-i18next'; -import { useDispatch } from 'react-redux'; +import { useDispatch, useSelector, shallowEqual } from 'react-redux'; import { renderInputField, normalizeMac } from '../../../../helpers/form'; import { @@ -25,6 +25,7 @@ const Form = ({ }) => { const { t } = useTranslation(); const dispatch = useDispatch(); + const dynamicLease = useSelector((store) => store.dhcp.leaseModalConfig, shallowEqual); const onClick = () => { reset(); @@ -87,7 +88,7 @@ const Form = ({ diff --git a/client/src/components/Settings/Dhcp/StaticLeases/Modal.js b/client/src/components/Settings/Dhcp/StaticLeases/Modal.js index 0baf487e..7a11cfce 100644 --- a/client/src/components/Settings/Dhcp/StaticLeases/Modal.js +++ b/client/src/components/Settings/Dhcp/StaticLeases/Modal.js @@ -2,7 +2,7 @@ import React from 'react'; import PropTypes from 'prop-types'; import { Trans, withTranslation } from 'react-i18next'; import ReactModal from 'react-modal'; -import { useDispatch } from 'react-redux'; +import { shallowEqual, useDispatch, useSelector } from 'react-redux'; import Form from './Form'; import { toggleLeaseModal } from '../../../../actions'; @@ -18,6 +18,9 @@ const Modal = ({ const dispatch = useDispatch(); const toggleModal = () => dispatch(toggleLeaseModal()); + const leaseInitialData = useSelector( + (state) => state.dhcp.leaseModalConfig, shallowEqual, + ) || {}; return (
dhcp_table_hostname, accessor: 'hostname', + minWidth: 230, Cell: cellWrap, }, { diff --git a/client/src/components/Settings/Dhcp/index.js b/client/src/components/Settings/Dhcp/index.js index bd3a45e3..a509ac49 100644 --- a/client/src/components/Settings/Dhcp/index.js +++ b/client/src/components/Settings/Dhcp/index.js @@ -188,8 +188,8 @@ const Dhcp = () => { const inputtedIPv4values = dhcp?.values?.v4?.gateway_ip && dhcp?.values?.v4?.subnet_mask; const isEmptyConfig = !Object.values(dhcp?.values?.v4 ?? {}).some(Boolean); - const disabledLeasesButton = dhcp?.syncErrors || interfaces?.syncErrors - || !isInterfaceIncludesIpv4 || isEmptyConfig || processingConfig || !inputtedIPv4values; + const disabledLeasesButton = Boolean(dhcp?.syncErrors || interfaces?.syncErrors + || !isInterfaceIncludesIpv4 || isEmptyConfig || processingConfig || !inputtedIPv4values); const cidr = inputtedIPv4values ? `${dhcp?.values?.v4?.gateway_ip}/${subnetMaskToBitMask(dhcp?.values?.v4?.subnet_mask)}` : ''; return <> @@ -260,7 +260,7 @@ const Dhcp = () => { >
- +
} diff --git a/client/src/components/Settings/Dns/Config/Form.js b/client/src/components/Settings/Dns/Config/Form.js index a2dd2bf9..52d94741 100644 --- a/client/src/components/Settings/Dns/Config/Form.js +++ b/client/src/components/Settings/Dns/Config/Form.js @@ -13,15 +13,11 @@ import { validateIpv4, validateIpv6, validateRequiredValue, + validateIp, } from '../../../../helpers/validators'; import { BLOCKING_MODES, FORM_NAME, UINT32_RANGE } from '../../../../helpers/constants'; const checkboxes = [ - { - name: 'edns_cs_enabled', - placeholder: 'edns_enable', - subtitle: 'edns_cs_desc', - }, { name: 'dnssec_enabled', placeholder: 'dnssec_enable', @@ -66,6 +62,8 @@ const Form = ({ const { t } = useTranslation(); const { blocking_mode, + edns_cs_enabled, + edns_cs_use_custom, } = useSelector((state) => state.form[FORM_NAME.BLOCKING_MODE].values ?? {}, shallowEqual); return @@ -92,6 +90,39 @@ const Form = ({ /> +
+
+ +
+
+
+
+ +
+ + {edns_cs_use_custom && ()} + +
{checkboxes.map(({ name, placeholder, subtitle }) =>
{ blocking_ipv4, blocking_ipv6, edns_cs_enabled, + edns_cs_use_custom, + edns_cs_custom_ip, dnssec_enabled, disable_ipv6, processingSetConfig, @@ -39,6 +41,8 @@ const Config = () => { edns_cs_enabled, disable_ipv6, dnssec_enabled, + edns_cs_use_custom, + edns_cs_custom_ip, }} onSubmit={handleFormSubmit} processing={processingSetConfig} diff --git a/client/src/reducers/dhcp.js b/client/src/reducers/dhcp.js index 4c2fe991..c0d6e8c0 100644 --- a/client/src/reducers/dhcp.js +++ b/client/src/reducers/dhcp.js @@ -124,10 +124,11 @@ const dhcp = handleActions( staticLeases: [], }), - [actions.toggleLeaseModal]: (state) => { + [actions.toggleLeaseModal]: (state, { payload }) => { const newState = { ...state, isModalOpen: !state.isModalOpen, + leaseModalConfig: payload, }; return newState; }, @@ -200,6 +201,7 @@ const dhcp = handleActions( leases: [], staticLeases: [], isModalOpen: false, + leaseModalConfig: undefined, dhcp_available: false, }, ); diff --git a/internal/dnsforward/config.go b/internal/dnsforward/config.go index f18e7513..008b2539 100644 --- a/internal/dnsforward/config.go +++ b/internal/dnsforward/config.go @@ -200,7 +200,7 @@ type FilteringConfig struct { // EDNSClientSubnet is the settings list for EDNS Client Subnet. type EDNSClientSubnet struct { // CustomIP for EDNS Client Subnet. - CustomIP string `yaml:"custom_ip"` + CustomIP netip.Addr `yaml:"custom_ip"` // Enabled defines if EDNS Client Subnet is enabled. Enabled bool `yaml:"enabled"` @@ -340,15 +340,8 @@ func (s *Server) createProxyConfig() (conf proxy.Config, err error) { } if srvConf.EDNSClientSubnet.UseCustom { - // TODO(s.chzhen): Add wrapper around netip.Addr. - var ip net.IP - ip, err = netutil.ParseIP(srvConf.EDNSClientSubnet.CustomIP) - if err != nil { - return conf, fmt.Errorf("edns: %w", err) - } - // TODO(s.chzhen): Use netip.Addr instead of net.IP inside dnsproxy. - conf.EDNSAddr = ip + conf.EDNSAddr = net.IP(srvConf.EDNSClientSubnet.CustomIP.AsSlice()) } if srvConf.CacheSize != 0 { @@ -377,7 +370,7 @@ func (s *Server) createProxyConfig() (conf proxy.Config, err error) { err = s.prepareTLS(&conf) if err != nil { - return conf, fmt.Errorf("validating tls: %w", err) + return proxy.Config{}, fmt.Errorf("validating tls: %w", err) } if c := srvConf.DNSCryptConfig; c.Enabled { @@ -388,7 +381,7 @@ func (s *Server) createProxyConfig() (conf proxy.Config, err error) { } if conf.UpstreamConfig == nil || len(conf.UpstreamConfig.Upstreams) == 0 { - return conf, errors.Error("no default upstream servers configured") + return proxy.Config{}, errors.Error("no default upstream servers configured") } return conf, nil diff --git a/internal/dnsforward/http.go b/internal/dnsforward/http.go index 0c8b6726..924a3675 100644 --- a/internal/dnsforward/http.go +++ b/internal/dnsforward/http.go @@ -23,26 +23,78 @@ import ( ) // jsonDNSConfig is the JSON representation of the DNS server configuration. +// +// TODO(s.chzhen): Split it into smaller pieces. Use aghalg.NullBool instead +// of *bool. type jsonDNSConfig struct { - Upstreams *[]string `json:"upstream_dns"` - UpstreamsFile *string `json:"upstream_dns_file"` - Bootstraps *[]string `json:"bootstrap_dns"` - ProtectionEnabled *bool `json:"protection_enabled"` - RateLimit *uint32 `json:"ratelimit"` - BlockingMode *BlockingMode `json:"blocking_mode"` - EDNSCSEnabled *bool `json:"edns_cs_enabled"` - DNSSECEnabled *bool `json:"dnssec_enabled"` - DisableIPv6 *bool `json:"disable_ipv6"` - UpstreamMode *string `json:"upstream_mode"` - CacheSize *uint32 `json:"cache_size"` - CacheMinTTL *uint32 `json:"cache_ttl_min"` - CacheMaxTTL *uint32 `json:"cache_ttl_max"` - CacheOptimistic *bool `json:"cache_optimistic"` - ResolveClients *bool `json:"resolve_clients"` - UsePrivateRDNS *bool `json:"use_private_ptr_resolvers"` - LocalPTRUpstreams *[]string `json:"local_ptr_upstreams"` - BlockingIPv4 net.IP `json:"blocking_ipv4"` - BlockingIPv6 net.IP `json:"blocking_ipv6"` + // Upstreams is the list of upstream DNS servers. + Upstreams *[]string `json:"upstream_dns"` + + // UpstreamsFile is the file containing upstream DNS servers. + UpstreamsFile *string `json:"upstream_dns_file"` + + // Bootstraps is the list of DNS servers resolving IP addresses of the + // upstream DoH/DoT resolvers. + Bootstraps *[]string `json:"bootstrap_dns"` + + // ProtectionEnabled defines if protection is enabled. + ProtectionEnabled *bool `json:"protection_enabled"` + + // RateLimit is the number of requests per second allowed per client. + RateLimit *uint32 `json:"ratelimit"` + + // BlockingMode defines the way blocked responses are constructed. + BlockingMode *BlockingMode `json:"blocking_mode"` + + // EDNSCSEnabled defines if EDNS Client Subnet is enabled. + EDNSCSEnabled *bool `json:"edns_cs_enabled"` + + // EDNSCSUseCustom defines if EDNSCSCustomIP should be used. + EDNSCSUseCustom *bool `json:"edns_cs_use_custom"` + + // DNSSECEnabled defines if DNSSEC is enabled. + DNSSECEnabled *bool `json:"dnssec_enabled"` + + // DisableIPv6 defines if IPv6 addresses should be dropped. + DisableIPv6 *bool `json:"disable_ipv6"` + + // UpstreamMode defines the way DNS requests are constructed. + UpstreamMode *string `json:"upstream_mode"` + + // CacheSize in bytes. + CacheSize *uint32 `json:"cache_size"` + + // CacheMinTTL is custom minimum TTL for cached DNS responses. + CacheMinTTL *uint32 `json:"cache_ttl_min"` + + // CacheMaxTTL is custom maximum TTL for cached DNS responses. + CacheMaxTTL *uint32 `json:"cache_ttl_max"` + + // CacheOptimistic defines if expired entries should be served. + CacheOptimistic *bool `json:"cache_optimistic"` + + // ResolveClients defines if clients IPs should be resolved into hostnames. + ResolveClients *bool `json:"resolve_clients"` + + // UsePrivateRDNS defines if privates DNS resolvers should be used. + UsePrivateRDNS *bool `json:"use_private_ptr_resolvers"` + + // LocalPTRUpstreams is the list of local private DNS resolvers. + LocalPTRUpstreams *[]string `json:"local_ptr_upstreams"` + + // BlockingIPv4 is custom IPv4 address for blocked A requests. + BlockingIPv4 net.IP `json:"blocking_ipv4"` + + // BlockingIPv6 is custom IPv6 address for blocked AAAA requests. + BlockingIPv6 net.IP `json:"blocking_ipv6"` + + // EDNSCSCustomIP is custom IP for EDNS Client Subnet. + EDNSCSCustomIP netip.Addr `json:"edns_cs_custom_ip"` + + // DefaultLocalPTRUpstreams is used to pass the addresses from + // systemResolvers to the front-end. It's not a pointer to the slice since + // there is no need to omit it while decoding from JSON. + DefaultLocalPTRUpstreams []string `json:"default_local_ptr_upstreams,omitempty"` } func (s *Server) getDNSConfig() (c *jsonDNSConfig) { @@ -57,7 +109,11 @@ func (s *Server) getDNSConfig() (c *jsonDNSConfig) { blockingIPv4 := s.conf.BlockingIPv4 blockingIPv6 := s.conf.BlockingIPv6 ratelimit := s.conf.Ratelimit + + customIP := s.conf.EDNSClientSubnet.CustomIP enableEDNSClientSubnet := s.conf.EDNSClientSubnet.Enabled + useCustom := s.conf.EDNSClientSubnet.UseCustom + enableDNSSEC := s.conf.EnableDNSSEC aaaaDisabled := s.conf.AAAADisabled cacheSize := s.conf.CacheSize @@ -74,46 +130,40 @@ func (s *Server) getDNSConfig() (c *jsonDNSConfig) { upstreamMode = "parallel" } - return &jsonDNSConfig{ - Upstreams: &upstreams, - UpstreamsFile: &upstreamFile, - Bootstraps: &bootstraps, - ProtectionEnabled: &protectionEnabled, - BlockingMode: &blockingMode, - BlockingIPv4: blockingIPv4, - BlockingIPv6: blockingIPv6, - RateLimit: &ratelimit, - EDNSCSEnabled: &enableEDNSClientSubnet, - DNSSECEnabled: &enableDNSSEC, - DisableIPv6: &aaaaDisabled, - CacheSize: &cacheSize, - CacheMinTTL: &cacheMinTTL, - CacheMaxTTL: &cacheMaxTTL, - CacheOptimistic: &cacheOptimistic, - UpstreamMode: &upstreamMode, - ResolveClients: &resolveClients, - UsePrivateRDNS: &usePrivateRDNS, - LocalPTRUpstreams: &localPTRUpstreams, - } -} - -func (s *Server) handleGetConfig(w http.ResponseWriter, r *http.Request) { defLocalPTRUps, err := s.filterOurDNSAddrs(s.sysResolvers.Get()) if err != nil { log.Debug("getting dns configuration: %s", err) } - resp := struct { - jsonDNSConfig - // DefautLocalPTRUpstreams is used to pass the addresses from - // systemResolvers to the front-end. It's not a pointer to the slice - // since there is no need to omit it while decoding from JSON. - DefautLocalPTRUpstreams []string `json:"default_local_ptr_upstreams,omitempty"` - }{ - jsonDNSConfig: *s.getDNSConfig(), - DefautLocalPTRUpstreams: defLocalPTRUps, + return &jsonDNSConfig{ + Upstreams: &upstreams, + UpstreamsFile: &upstreamFile, + Bootstraps: &bootstraps, + ProtectionEnabled: &protectionEnabled, + BlockingMode: &blockingMode, + BlockingIPv4: blockingIPv4, + BlockingIPv6: blockingIPv6, + RateLimit: &ratelimit, + EDNSCSCustomIP: customIP, + EDNSCSEnabled: &enableEDNSClientSubnet, + EDNSCSUseCustom: &useCustom, + DNSSECEnabled: &enableDNSSEC, + DisableIPv6: &aaaaDisabled, + CacheSize: &cacheSize, + CacheMinTTL: &cacheMinTTL, + CacheMaxTTL: &cacheMaxTTL, + CacheOptimistic: &cacheOptimistic, + UpstreamMode: &upstreamMode, + ResolveClients: &resolveClients, + UsePrivateRDNS: &usePrivateRDNS, + LocalPTRUpstreams: &localPTRUpstreams, + DefaultLocalPTRUpstreams: defLocalPTRUps, } +} +// handleGetConfig handles requests to the GET /control/dns_info endpoint. +func (s *Server) handleGetConfig(w http.ResponseWriter, r *http.Request) { + resp := s.getDNSConfig() _ = aghhttp.WriteJSONResponse(w, r, resp) } @@ -204,6 +254,7 @@ func (req *jsonDNSConfig) checkCacheTTL() bool { return min <= max } +// handleSetConfig handles requests to the POST /control/dns_config endpoint. func (s *Server) handleSetConfig(w http.ResponseWriter, r *http.Request) { req := &jsonDNSConfig{} err := json.NewDecoder(r.Body).Decode(req) @@ -231,8 +282,8 @@ func (s *Server) handleSetConfig(w http.ResponseWriter, r *http.Request) { } } -// setConfigRestartable sets the server parameters. shouldRestart is true if -// the server should be restarted to apply changes. +// setConfig sets the server parameters. shouldRestart is true if the server +// should be restarted to apply changes. func (s *Server) setConfig(dc *jsonDNSConfig) (shouldRestart bool) { s.serverLock.Lock() defer s.serverLock.Unlock() @@ -250,6 +301,10 @@ func (s *Server) setConfig(dc *jsonDNSConfig) (shouldRestart bool) { s.conf.FastestAddr = *dc.UpstreamMode == "fastest_addr" } + if dc.EDNSCSUseCustom != nil && *dc.EDNSCSUseCustom { + s.conf.EDNSClientSubnet.CustomIP = dc.EDNSCSCustomIP + } + setIfNotNil(&s.conf.ProtectionEnabled, dc.ProtectionEnabled) setIfNotNil(&s.conf.EnableDNSSEC, dc.DNSSECEnabled) setIfNotNil(&s.conf.AAAADisabled, dc.DisableIPv6) @@ -281,6 +336,7 @@ func (s *Server) setConfigRestartable(dc *jsonDNSConfig) (shouldRestart bool) { setIfNotNil(&s.conf.UpstreamDNSFileName, dc.UpstreamsFile), setIfNotNil(&s.conf.BootstrapDNS, dc.Bootstraps), setIfNotNil(&s.conf.EDNSClientSubnet.Enabled, dc.EDNSCSEnabled), + setIfNotNil(&s.conf.EDNSClientSubnet.UseCustom, dc.EDNSCSUseCustom), setIfNotNil(&s.conf.CacheSize, dc.CacheSize), setIfNotNil(&s.conf.CacheMinTTL, dc.CacheMinTTL), setIfNotNil(&s.conf.CacheMaxTTL, dc.CacheMaxTTL), diff --git a/internal/dnsforward/http_test.go b/internal/dnsforward/http_test.go index ef2228c1..144568d3 100644 --- a/internal/dnsforward/http_test.go +++ b/internal/dnsforward/http_test.go @@ -181,6 +181,12 @@ func TestDNSForwardHTTP_handleSetConfig(t *testing.T) { }, { name: "edns_cs_enabled", wantSet: "", + }, { + name: "edns_cs_use_custom", + wantSet: "", + }, { + name: "edns_cs_use_custom_bad_ip", + wantSet: "decoding request: ParseAddr(\"bad.ip\"): unexpected character (at \"bad.ip\")", }, { name: "dnssec_enabled", wantSet: "", @@ -222,16 +228,20 @@ func TestDNSForwardHTTP_handleSetConfig(t *testing.T) { Req json.RawMessage `json:"req"` Want json.RawMessage `json:"want"` } - loadTestData(t, t.Name()+jsonExt, &data) + + testData := t.Name() + jsonExt + loadTestData(t, testData, &data) for _, tc := range testCases { + // NOTE: Do not use require.Contains, because the size of the data + // prevents it from printing a meaningful error message. caseData, ok := data[tc.name] - require.True(t, ok) + require.Truef(t, ok, "%q does not contain test data for test case %s", testData, tc.name) t.Run(tc.name, func(t *testing.T) { t.Cleanup(func() { s.conf = defaultConf - s.conf.FilteringConfig.EDNSClientSubnet.Enabled = false + s.conf.FilteringConfig.EDNSClientSubnet = &EDNSClientSubnet{} }) rBody := io.NopCloser(bytes.NewReader(caseData.Req)) diff --git a/internal/dnsforward/testdata/TestDNSForwardHTTP_handleGetConfig.json b/internal/dnsforward/testdata/TestDNSForwardHTTP_handleGetConfig.json index 3ac6f2f5..fe2c5666 100644 --- a/internal/dnsforward/testdata/TestDNSForwardHTTP_handleGetConfig.json +++ b/internal/dnsforward/testdata/TestDNSForwardHTTP_handleGetConfig.json @@ -26,7 +26,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" }, "fastest_addr": { "upstream_dns": [ @@ -55,7 +57,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" }, "parallel": { "upstream_dns": [ @@ -84,6 +88,8 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } } diff --git a/internal/dnsforward/testdata/TestDNSForwardHTTP_handleSetConfig.json b/internal/dnsforward/testdata/TestDNSForwardHTTP_handleSetConfig.json index f55359a9..ca8c963a 100644 --- a/internal/dnsforward/testdata/TestDNSForwardHTTP_handleSetConfig.json +++ b/internal/dnsforward/testdata/TestDNSForwardHTTP_handleSetConfig.json @@ -33,7 +33,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "bootstraps": { @@ -66,7 +68,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "blocking_mode_good": { @@ -100,7 +104,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "blocking_mode_bad": { @@ -134,7 +140,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "ratelimit": { @@ -168,7 +176,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "edns_cs_enabled": { @@ -202,7 +212,85 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" + } + }, + "edns_cs_use_custom": { + "req": { + "edns_cs_enabled": true, + "edns_cs_use_custom": true, + "edns_cs_custom_ip": "1.2.3.4" + }, + "want": { + "upstream_dns": [ + "8.8.8.8:53", + "8.8.4.4:53" + ], + "upstream_dns_file": "", + "bootstrap_dns": [ + "9.9.9.10", + "149.112.112.10", + "2620:fe::10", + "2620:fe::fe:10" + ], + "protection_enabled": true, + "ratelimit": 0, + "blocking_mode": "default", + "blocking_ipv4": "", + "blocking_ipv6": "", + "edns_cs_enabled": true, + "dnssec_enabled": false, + "disable_ipv6": false, + "upstream_mode": "", + "cache_size": 0, + "cache_ttl_min": 0, + "cache_ttl_max": 0, + "cache_optimistic": false, + "resolve_clients": false, + "use_private_ptr_resolvers": false, + "local_ptr_upstreams": [], + "edns_cs_use_custom": true, + "edns_cs_custom_ip": "1.2.3.4" + } + }, + "edns_cs_use_custom_bad_ip": { + "req": { + "edns_cs_enabled": true, + "edns_cs_use_custom": true, + "edns_cs_custom_ip": "bad.ip" + }, + "want": { + "upstream_dns": [ + "8.8.8.8:53", + "8.8.4.4:53" + ], + "upstream_dns_file": "", + "bootstrap_dns": [ + "9.9.9.10", + "149.112.112.10", + "2620:fe::10", + "2620:fe::fe:10" + ], + "protection_enabled": true, + "ratelimit": 0, + "blocking_mode": "default", + "blocking_ipv4": "", + "blocking_ipv6": "", + "edns_cs_enabled": false, + "dnssec_enabled": false, + "disable_ipv6": false, + "upstream_mode": "", + "cache_size": 0, + "cache_ttl_min": 0, + "cache_ttl_max": 0, + "cache_optimistic": false, + "resolve_clients": false, + "use_private_ptr_resolvers": false, + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "dnssec_enabled": { @@ -236,7 +324,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "cache_size": { @@ -270,7 +360,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "upstream_mode_parallel": { @@ -304,7 +396,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "upstream_mode_fastest_addr": { @@ -338,7 +432,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "upstream_dns_bad": { @@ -374,7 +470,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "bootstraps_bad": { @@ -410,7 +508,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "cache_bad_ttl": { @@ -445,7 +545,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "upstream_mode_bad": { @@ -479,7 +581,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "local_ptr_upstreams_good": { @@ -517,7 +621,9 @@ "use_private_ptr_resolvers": false, "local_ptr_upstreams": [ "123.123.123.123" - ] + ], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "local_ptr_upstreams_bad": { @@ -554,7 +660,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } }, "local_ptr_upstreams_null": { @@ -588,7 +696,9 @@ "cache_optimistic": false, "resolve_clients": false, "use_private_ptr_resolvers": false, - "local_ptr_upstreams": [] + "local_ptr_upstreams": [], + "edns_cs_use_custom": false, + "edns_cs_custom_ip": "" } } } diff --git a/internal/home/config.go b/internal/home/config.go index 6aad27b6..30dcb69d 100644 --- a/internal/home/config.go +++ b/internal/home/config.go @@ -286,7 +286,7 @@ var config = &configuration{ CacheSize: 4 * 1024 * 1024, EDNSClientSubnet: &dnsforward.EDNSClientSubnet{ - CustomIP: "", + CustomIP: netip.Addr{}, Enabled: false, UseCustom: false, }, diff --git a/openapi/CHANGELOG.md b/openapi/CHANGELOG.md index 20dde662..3b43140e 100644 --- a/openapi/CHANGELOG.md +++ b/openapi/CHANGELOG.md @@ -4,6 +4,18 @@ ## v0.108.0: API changes +## v0.107.27: API changes + +### The new optional fields `"edns_cs_use_custom"` and `"edns_cs_custom_ip"` in `DNSConfig` + +* The new optional fields `"edns_cs_use_custom"` and `"edns_cs_custom_ip"` in + `POST /control/dns_config` method makes AdGuard Home use or not use the + custom IP for EDNS Client Subnet. + +* The new optional fields `"edns_cs_use_custom"` and `"edns_cs_custom_ip"` in + `GET /control/dns_info` method are set if AdGuard Home uses custom IP for + EDNS Client Subnet. + ## v0.107.23: API changes diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 0bbac1e0..2ec4d858 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -1254,7 +1254,7 @@ 'example': 'en' 'DNSConfig': 'type': 'object' - 'description': 'Query log configuration' + 'description': 'DNS server configuration' 'properties': 'bootstrap_dns': 'type': 'array' @@ -1280,8 +1280,6 @@ 'type': 'string' 'protection_enabled': 'type': 'boolean' - 'dhcp_available': - 'type': 'boolean' 'ratelimit': 'type': 'integer' 'blocking_mode': @@ -1298,6 +1296,10 @@ 'type': 'string' 'edns_cs_enabled': 'type': 'boolean' + 'edns_cs_use_custom': + 'type': 'boolean' + 'edns_cs_custom_ip': + 'type': 'string' 'disable_ipv6': 'type': 'boolean' 'dnssec_enabled': diff --git a/scripts/README.md b/scripts/README.md index 5b7475bb..4821849f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -178,22 +178,28 @@ manifest file templates, and helper scripts. ### Usage - * `npm install`: install dependencies. Run this first. - * `npm run locales:download`: download and save all translations. - * `npm run locales:upload`: upload the base `en` locale. - * `npm run locales:summary`: show the current locales summary. - * `npm run locales:unused`: show the list of unused strings. + * `go run main.go help`: print usage. + + * `go run main.go download [-n ]`: download and save all translations. + `n` is optional flag where count is a number of concurrent downloads. + + * `go run main.go upload`: upload the base `en` locale. + + * `go run main.go summary`: show the current locales summary. + + * `go run main.go unused`: show the list of unused strings. After the download you'll find the output locales in the `client/src/__locales/` directory. Optional environment: - * `SLEEP_TIME`: set the sleep time between downloads for `locales:download`, - in milliseconds. The default is 250 ms. + * `UPLOAD_LANGUAGE`: set an alternative language for `upload`. - * `UPLOAD_LANGUAGE`: set an alternative language for `locales:upload` to - upload. + * `TWOSKY_URI`: set an alternative URL for `download` or `upload`. + + * `TWOSKY_PROJECT_ID`: set an alternative project ID for `download` or + `upload`. diff --git a/scripts/make/go-lint.sh b/scripts/make/go-lint.sh index 3216b40a..35200f49 100644 --- a/scripts/make/go-lint.sh +++ b/scripts/make/go-lint.sh @@ -182,6 +182,7 @@ run_linter gocyclo --over 10\ ./internal/version/\ ./scripts/blocked-services/\ ./scripts/vetted-filters/\ + ./scripts/translations/\ ./main.go\ ; diff --git a/scripts/translations/.gitignore b/scripts/translations/.gitignore deleted file mode 100644 index 3c3629e6..00000000 --- a/scripts/translations/.gitignore +++ /dev/null @@ -1 +0,0 @@ -node_modules diff --git a/scripts/translations/count.js b/scripts/translations/count.js deleted file mode 100644 index c3369f96..00000000 --- a/scripts/translations/count.js +++ /dev/null @@ -1,41 +0,0 @@ -const path = require('path'); -const twoskyConfig = require('../../.twosky.json')[0]; - -const {languages} = twoskyConfig; -const LOCALES_DIR = '../../client/src/__locales'; -const LOCALES_LIST = Object.keys(languages); -const BASE_FILE = 'en.json'; - -const main = () => { - const pathToBaseFile = path.join(LOCALES_DIR, BASE_FILE); - const baseLanguageJson = require(pathToBaseFile); - - const summary = {}; - - LOCALES_LIST.forEach((locale) => { - const pathToFile = path.join(LOCALES_DIR, `${locale}.json`); - if (pathToFile === pathToBaseFile) { - return; - } - - let total = 0; - let translated = 0; - - const languageJson = require(pathToFile); - for (let key in baseLanguageJson) { - total += 1; - if (key in languageJson) { - translated += 1; - } - } - - summary[locale] = Math.round(translated / total * 10000) / 100; - }); - - console.log('Translations summary:'); - for (let key in summary) { - console.log(`${key}, translated ${summary[key]}%`); - } -} - -main(); diff --git a/scripts/translations/download.js b/scripts/translations/download.js deleted file mode 100644 index 6cb65072..00000000 --- a/scripts/translations/download.js +++ /dev/null @@ -1,125 +0,0 @@ -// TODO(a.garipov): Rewrite this in Go; add better concurrency controls; add -// features for easier maintenance. - -const fs = require('fs'); -const path = require('path'); -const requestPromise = require('request-promise'); -const twoskyConfig = require('../../.twosky.json')[0]; - -const { project_id: TWOSKY_PROJECT_ID, languages } = twoskyConfig; -const LOCALES_DIR = '../../client/src/__locales'; -const LOCALES_LIST = Object.keys(languages); -const BASE_FILE = 'en.json'; -const TWOSKY_URI = process.env.TWOSKY_URI; - -/** - * Prepare params to get translations from twosky - * @param {string} locale language shortcut - * @param {object} twosky config twosky - */ -const getRequestUrl = (locale, url, projectId) => { - return `${url}/download?format=json&language=${locale}&filename=${BASE_FILE}&project=${projectId}`; -}; - -/** - * Promise wrapper for writing in file - * @param {string} filename - * @param {any} body - */ -function writeInFile(filename, body) { - let normalizedBody = removeEmpty(JSON.parse(body)); - - return new Promise((resolve, reject) => { - if (typeof normalizedBody !== 'string') { - try { - normalizedBody = JSON.stringify(normalizedBody, null, 4) + '\n'; // eslint-disable-line - } catch (err) { - reject(err); - } - } - - fs.writeFile(filename, normalizedBody, (err) => { - if (err) reject(err); - resolve('Ok'); - }); - }); -} - -/** - * Clear initial from empty value keys - * @param {object} initialObject - */ -function removeEmpty(initialObject) { - let processedObject = {}; - Object.keys(initialObject).forEach(prop => { - if (initialObject[prop]) { - processedObject[prop] = initialObject[prop]; - } - }); - return processedObject; -} - -/** - * Request twosky - * @param {string} url - * @param {string} locale - */ -const request = (url, locale) => ( - requestPromise.get(url) - .then((res) => { - if (res.length) { - const pathToFile = path.join(LOCALES_DIR, `${locale}.json`); - return writeInFile(pathToFile, res); - } - return null; - }) - .then((res) => { - let result = locale; - result += res ? ' - OK' : ' - Empty'; - return result; - }) - .catch((err) => { - console.log(err); - return `${locale} - Not OK`; - })); - -/** - * Sleep. - * @param {number} ms - */ -const sleep = (ms) => new Promise((resolve) => { - setTimeout(resolve, ms); -}); - -/** - * Download locales - */ -const download = async () => { - const locales = LOCALES_LIST; - - if (!TWOSKY_URI) { - console.error('No credentials'); - return; - } - - const requests = []; - for (let i = 0; i < locales.length; i++) { - const locale = locales[i]; - const url = getRequestUrl(locale, TWOSKY_URI, TWOSKY_PROJECT_ID); - requests.push(request(url, locale)); - - // Don't request the Crowdin API too aggressively to prevent spurious - // 400 errors. - const sleepTime = process.env.SLEEP_TIME || 250; - await sleep(sleepTime); - } - - Promise - .all(requests) - .then((res) => { - res.forEach(item => console.log(item)); - }) - .catch(err => console.log(err)); -}; - -download(); diff --git a/scripts/translations/main.go b/scripts/translations/main.go new file mode 100644 index 00000000..1922f614 --- /dev/null +++ b/scripts/translations/main.go @@ -0,0 +1,464 @@ +// translations downloads translations, uploads translations, prints summary +// for translations, prints unused strings. +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/AdguardTeam/AdGuardHome/internal/aghio" + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/log" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" +) + +const ( + twoskyConfFile = "./.twosky.json" + localesDir = "./client/src/__locales" + defaultBaseFile = "en.json" + defaultProjectID = "home" + srcDir = "./client/src" + twoskyURI = "https://twosky.int.agrd.dev/api/v1" + + readLimit = 1 * 1024 * 1024 +) + +// langCode is a language code. +type langCode string + +// languages is a map, where key is language code and value is display name. +type languages map[langCode]string + +// textlabel is a text label of localization. +type textLabel string + +// locales is a map, where key is text label and value is translation. +type locales map[textLabel]string + +func main() { + if len(os.Args) == 1 { + usage("need a command") + } + + if os.Args[1] == "help" { + usage("") + } + + uriStr := os.Getenv("TWOSKY_URI") + if uriStr == "" { + uriStr = twoskyURI + } + + uri, err := url.Parse(uriStr) + check(err) + + projectID := os.Getenv("TWOSKY_PROJECT_ID") + if projectID == "" { + projectID = defaultProjectID + } + + conf, err := readTwoskyConf() + check(err) + + switch os.Args[1] { + case "summary": + err = summary(conf.Languages) + check(err) + case "download": + err = download(uri, projectID, conf.Languages) + check(err) + case "unused": + err = unused() + check(err) + case "upload": + err = upload(uri, projectID, conf.BaseLangcode) + check(err) + default: + usage("unknown command") + } +} + +// check is a simple error-checking helper for scripts. +func check(err error) { + if err != nil { + panic(err) + } +} + +// usage prints usage. If addStr is not empty print addStr and exit with code +// 1, otherwise exit with code 0. +func usage(addStr string) { + const usageStr = `Usage: go run main.go [] +Commands: + help + Print usage. + summary + Print summary. + download [-n ] + Download translations. count is a number of concurrent downloads. + unused + Print unused strings. + upload + Upload translations.` + + if addStr != "" { + fmt.Printf("%s\n%s\n", addStr, usageStr) + + os.Exit(1) + } + + fmt.Println(usageStr) + + os.Exit(0) +} + +// twoskyConf is the configuration structure for localization. +type twoskyConf struct { + Languages languages `json:"languages"` + ProjectID string `json:"project_id"` + BaseLangcode langCode `json:"base_locale"` + LocalizableFiles []string `json:"localizable_files"` +} + +// readTwoskyConf returns configuration. +func readTwoskyConf() (t twoskyConf, err error) { + b, err := os.ReadFile(twoskyConfFile) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return twoskyConf{}, err + } + + var tsc []twoskyConf + err = json.Unmarshal(b, &tsc) + if err != nil { + err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err) + + return twoskyConf{}, err + } + + if len(tsc) == 0 { + err = fmt.Errorf("%q is empty", twoskyConfFile) + + return twoskyConf{}, err + } + + conf := tsc[0] + + for _, lang := range conf.Languages { + if lang == "" { + return twoskyConf{}, errors.Error("language is empty") + } + } + + return conf, nil +} + +// readLocales reads file with name fn and returns a map, where key is text +// label and value is localization. +func readLocales(fn string) (loc locales, err error) { + b, err := os.ReadFile(fn) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + loc = make(locales) + err = json.Unmarshal(b, &loc) + if err != nil { + err = fmt.Errorf("unmarshalling %q: %w", fn, err) + + return nil, err + } + + return loc, nil +} + +// summary prints summary for translations. +func summary(langs languages) (err error) { + basePath := filepath.Join(localesDir, defaultBaseFile) + baseLoc, err := readLocales(basePath) + if err != nil { + return fmt.Errorf("summary: %w", err) + } + + size := float64(len(baseLoc)) + + keys := maps.Keys(langs) + slices.Sort(keys) + + for _, lang := range keys { + name := filepath.Join(localesDir, string(lang)+".json") + if name == basePath { + continue + } + + var loc locales + loc, err = readLocales(name) + if err != nil { + return fmt.Errorf("summary: reading locales: %w", err) + } + + f := float64(len(loc)) * 100 / size + + fmt.Printf("%s\t %6.2f %%\n", lang, f) + } + + return nil +} + +// download and save all translations. uri is the base URL. projectID is the +// name of the project. +func download(uri *url.URL, projectID string, langs languages) (err error) { + var numWorker int + + flagSet := flag.NewFlagSet("download", flag.ExitOnError) + flagSet.Usage = func() { + usage("download command error") + } + flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads") + + err = flagSet.Parse(os.Args[2:]) + if err != nil { + // Don't wrap the error since there is exit on error. + return err + } + + if numWorker < 1 { + usage("count must be positive") + } + + downloadURI := uri.JoinPath("download") + + client := &http.Client{ + Timeout: 10 * time.Second, + } + + var wg sync.WaitGroup + uriCh := make(chan *url.URL, len(langs)) + + for i := 0; i < numWorker; i++ { + wg.Add(1) + go downloadWorker(&wg, client, uriCh) + } + + for lang := range langs { + uri = translationURL(downloadURI, defaultBaseFile, projectID, lang) + + uriCh <- uri + } + + close(uriCh) + wg.Wait() + + return nil +} + +// downloadWorker downloads translations by received urls and saves them. +func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.URL) { + defer wg.Done() + + for uri := range uriCh { + data, err := getTranslation(client, uri.String()) + if err != nil { + log.Error("download worker: getting translation: %s", err) + + continue + } + + q := uri.Query() + code := q.Get("language") + + name := filepath.Join(localesDir, code+".json") + err = os.WriteFile(name, data, 0o664) + if err != nil { + log.Error("download worker: writing file: %s", err) + + continue + } + + fmt.Println(name) + } +} + +// getTranslation returns received translation data or error. +func getTranslation(client *http.Client, url string) (data []byte, err error) { + resp, err := client.Get(url) + if err != nil { + return nil, fmt.Errorf("requesting: %w", err) + } + + defer log.OnCloserError(resp.Body, log.ERROR) + + if resp.StatusCode != http.StatusOK { + err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode)) + + return nil, err + } + + limitReader, err := aghio.LimitReader(resp.Body, readLimit) + if err != nil { + err = fmt.Errorf("limit reading: %w", err) + + return nil, err + } + + data, err = io.ReadAll(limitReader) + if err != nil { + err = fmt.Errorf("reading all: %w", err) + + return nil, err + } + + return data, nil +} + +// translationURL returns a new url.URL with provided query parameters. +func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) { + uri = &url.URL{} + *uri = *oldURL + + q := uri.Query() + q.Set("format", "json") + q.Set("filename", baseFile) + q.Set("project", projectID) + q.Set("language", string(lang)) + + uri.RawQuery = q.Encode() + + return uri +} + +// unused prints unused text labels. +func unused() (err error) { + fileNames := []string{} + basePath := filepath.Join(localesDir, defaultBaseFile) + baseLoc, err := readLocales(basePath) + if err != nil { + return fmt.Errorf("unused: %w", err) + } + + locDir := filepath.Clean(localesDir) + + err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error { + if err != nil { + log.Info("accessing a path %q: %s", name, err) + + return nil + } + + if info.IsDir() { + return nil + } + + if strings.HasPrefix(name, locDir) { + return nil + } + + ext := filepath.Ext(name) + if ext == ".js" || ext == ".json" { + fileNames = append(fileNames, name) + } + + return nil + }) + + if err != nil { + return fmt.Errorf("filepath walking %q: %w", srcDir, err) + } + + err = removeUnused(fileNames, baseLoc) + + return errors.Annotate(err, "removing unused: %w") +} + +func removeUnused(fileNames []string, loc locales) (err error) { + knownUsed := []textLabel{ + "blocking_mode_refused", + "blocking_mode_nxdomain", + "blocking_mode_custom_ip", + } + + for _, v := range knownUsed { + delete(loc, v) + } + + for _, fn := range fileNames { + var buf []byte + buf, err = os.ReadFile(fn) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return err + } + + for k := range loc { + if bytes.Contains(buf, []byte(k)) { + delete(loc, k) + } + } + } + + printUnused(loc) + + return nil +} + +// printUnused text labels to stdout. +func printUnused(loc locales) { + keys := maps.Keys(loc) + slices.Sort(keys) + + for _, v := range keys { + fmt.Println(v) + } +} + +// upload base translation. uri is the base URL. projectID is the name of the +// project. baseLang is the base language code. +func upload(uri *url.URL, projectID string, baseLang langCode) (err error) { + uploadURI := uri.JoinPath("upload") + + lang := baseLang + + langStr := os.Getenv("UPLOAD_LANGUAGE") + if langStr != "" { + lang = langCode(langStr) + } + + basePath := filepath.Join(localesDir, defaultBaseFile) + b, err := os.ReadFile(basePath) + if err != nil { + return fmt.Errorf("upload: %w", err) + } + + var buf bytes.Buffer + buf.Write(b) + + uri = translationURL(uploadURI, defaultBaseFile, projectID, lang) + + var client http.Client + resp, err := client.Post(uri.String(), "application/json", &buf) + if err != nil { + return fmt.Errorf("upload: client post: %w", err) + } + + defer func() { + err = errors.WithDeferred(err, resp.Body.Close()) + }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode)) + } + + return nil +} diff --git a/scripts/translations/package-lock.json b/scripts/translations/package-lock.json deleted file mode 100644 index 42f5d23a..00000000 --- a/scripts/translations/package-lock.json +++ /dev/null @@ -1,838 +0,0 @@ -{ - "name": "translations", - "version": "0.2.0", - "lockfileVersion": 2, - "requires": true, - "packages": { - "": { - "version": "0.2.0", - "dependencies": { - "request": "^2.88.0", - "request-promise": "^4.2.2" - } - }, - "node_modules/ajv": { - "version": "6.5.5", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.5.5.tgz", - "integrity": "sha512-7q7gtRQDJSyuEHjuVgHoUa2VuemFiCMrfQc9Tc08XTAc4Zj/5U1buQJ0HU6i7fKjXU09SVgSmxa4sLvuvS8Iyg==", - "dependencies": { - "fast-deep-equal": "^2.0.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "node_modules/asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "dependencies": { - "safer-buffer": "~2.1.0" - } - }, - "node_modules/assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", - "engines": { - "node": ">=0.8" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" - }, - "node_modules/aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", - "engines": { - "node": "*" - } - }, - "node_modules/aws4": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", - "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" - }, - "node_modules/bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "dependencies": { - "tweetnacl": "^0.14.3" - } - }, - "node_modules/bluebird": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz", - "integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw==" - }, - "node_modules/caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" - }, - "node_modules/combined-stream": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", - "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, - "node_modules/dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "dependencies": { - "assert-plus": "^1.0.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/ecc-jsbn": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "dependencies": { - "jsbn": "~0.1.0", - "safer-buffer": "^2.1.0" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "node_modules/extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", - "engines": [ - "node >=0.6.0" - ] - }, - "node_modules/fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" - }, - "node_modules/forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", - "engines": { - "node": "*" - } - }, - "node_modules/form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 0.12" - } - }, - "node_modules/getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "dependencies": { - "assert-plus": "^1.0.0" - } - }, - "node_modules/har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", - "engines": { - "node": ">=4" - } - }, - "node_modules/har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", - "dependencies": { - "ajv": "^6.5.5", - "har-schema": "^2.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "dependencies": { - "assert-plus": "^1.0.0", - "jsprim": "^1.2.2", - "sshpk": "^1.7.0" - }, - "engines": { - "node": ">=0.8", - "npm": ">=1.3.7" - } - }, - "node_modules/is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" - }, - "node_modules/isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" - }, - "node_modules/jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" - }, - "node_modules/json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" - }, - "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" - }, - "node_modules/json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" - }, - "node_modules/jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "engines": [ - "node >=0.6.0" - ], - "dependencies": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.2.3", - "verror": "1.10.0" - } - }, - "node_modules/lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" - }, - "node_modules/mime-db": { - "version": "1.37.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", - "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.21", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", - "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", - "dependencies": { - "mime-db": "~1.37.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/oauth-sign": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", - "engines": { - "node": "*" - } - }, - "node_modules/performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" - }, - "node_modules/psl": { - "version": "1.1.29", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz", - "integrity": "sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ==" - }, - "node_modules/punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" - }, - "node_modules/qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", - "dependencies": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.0", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - }, - "engines": { - "node": ">= 4" - } - }, - "node_modules/request-promise": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.2.tgz", - "integrity": "sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=", - "dependencies": { - "bluebird": "^3.5.0", - "request-promise-core": "1.1.1", - "stealthy-require": "^1.1.0", - "tough-cookie": ">=2.3.3" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/request-promise-core": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz", - "integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=", - "dependencies": { - "lodash": "^4.13.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "node_modules/sshpk": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.15.2.tgz", - "integrity": "sha512-Ra/OXQtuh0/enyl4ETZAfTaeksa6BXks5ZcjpSUNrjBr0DvrJKX+1fsKDPpT9TBXgHAFsa4510aNVgI8g/+SzA==", - "dependencies": { - "asn1": "~0.2.3", - "assert-plus": "^1.0.0", - "bcrypt-pbkdf": "^1.0.0", - "dashdash": "^1.12.0", - "ecc-jsbn": "~0.1.1", - "getpass": "^0.1.1", - "jsbn": "~0.1.0", - "safer-buffer": "^2.0.2", - "tweetnacl": "~0.14.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/stealthy-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", - "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", - "dependencies": { - "psl": "^1.1.24", - "punycode": "^1.4.1" - }, - "engines": { - "node": ">=0.8" - } - }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, - "node_modules/tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, - "node_modules/uri-js": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/uri-js/node_modules/punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "engines": { - "node": ">=6" - } - }, - "node_modules/uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "bin": { - "uuid": "bin/uuid" - } - }, - "node_modules/verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "engines": [ - "node >=0.6.0" - ], - "dependencies": { - "assert-plus": "^1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "^1.2.0" - } - } - }, - "dependencies": { - "ajv": { - "version": "6.5.5", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.5.5.tgz", - "integrity": "sha512-7q7gtRQDJSyuEHjuVgHoUa2VuemFiCMrfQc9Tc08XTAc4Zj/5U1buQJ0HU6i7fKjXU09SVgSmxa4sLvuvS8Iyg==", - "requires": { - "fast-deep-equal": "^2.0.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "requires": { - "safer-buffer": "~2.1.0" - } - }, - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - }, - "asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" - }, - "aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" - }, - "aws4": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", - "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" - }, - "bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "requires": { - "tweetnacl": "^0.14.3" - } - }, - "bluebird": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz", - "integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw==" - }, - "caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" - }, - "combined-stream": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", - "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", - "requires": { - "delayed-stream": "~1.0.0" - } - }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, - "dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "requires": { - "assert-plus": "^1.0.0" - } - }, - "delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" - }, - "ecc-jsbn": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "requires": { - "jsbn": "~0.1.0", - "safer-buffer": "^2.1.0" - } - }, - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" - }, - "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" - }, - "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" - }, - "forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" - }, - "form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - }, - "getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "requires": { - "assert-plus": "^1.0.0" - } - }, - "har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" - }, - "har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", - "requires": { - "ajv": "^6.5.5", - "har-schema": "^2.0.0" - } - }, - "http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "requires": { - "assert-plus": "^1.0.0", - "jsprim": "^1.2.2", - "sshpk": "^1.7.0" - } - }, - "is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" - }, - "isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" - }, - "jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" - }, - "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" - }, - "json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" - }, - "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "requires": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.2.3", - "verror": "1.10.0" - } - }, - "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" - }, - "mime-db": { - "version": "1.37.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", - "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==" - }, - "mime-types": { - "version": "2.1.21", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", - "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", - "requires": { - "mime-db": "~1.37.0" - } - }, - "oauth-sign": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" - }, - "performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" - }, - "psl": { - "version": "1.1.29", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz", - "integrity": "sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ==" - }, - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" - }, - "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" - }, - "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", - "requires": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.0", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - } - }, - "request-promise": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.2.tgz", - "integrity": "sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=", - "requires": { - "bluebird": "^3.5.0", - "request-promise-core": "1.1.1", - "stealthy-require": "^1.1.0", - "tough-cookie": ">=2.3.3" - } - }, - "request-promise-core": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz", - "integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=", - "requires": { - "lodash": "^4.13.1" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "sshpk": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.15.2.tgz", - "integrity": "sha512-Ra/OXQtuh0/enyl4ETZAfTaeksa6BXks5ZcjpSUNrjBr0DvrJKX+1fsKDPpT9TBXgHAFsa4510aNVgI8g/+SzA==", - "requires": { - "asn1": "~0.2.3", - "assert-plus": "^1.0.0", - "bcrypt-pbkdf": "^1.0.0", - "dashdash": "^1.12.0", - "ecc-jsbn": "~0.1.1", - "getpass": "^0.1.1", - "jsbn": "~0.1.0", - "safer-buffer": "^2.0.2", - "tweetnacl": "~0.14.0" - } - }, - "stealthy-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", - "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" - }, - "tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", - "requires": { - "psl": "^1.1.24", - "punycode": "^1.4.1" - } - }, - "tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "requires": { - "safe-buffer": "^5.0.1" - } - }, - "tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, - "uri-js": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", - "requires": { - "punycode": "^2.1.0" - }, - "dependencies": { - "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" - } - } - }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" - }, - "verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "requires": { - "assert-plus": "^1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "^1.2.0" - } - } - } -} diff --git a/scripts/translations/package.json b/scripts/translations/package.json deleted file mode 100644 index 110db720..00000000 --- a/scripts/translations/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "translations", - "version": "0.2.0", - "scripts": { - "locales:download": "TWOSKY_URI=https://twosky.int.agrd.dev/api/v1 TWOSKY_PROJECT_ID=home node download.js ; node count.js", - "locales:upload": "TWOSKY_URI=https://twosky.int.agrd.dev/api/v1 TWOSKY_PROJECT_ID=home node upload.js", - "locales:summary": "node count.js", - "locales:unused": "node unused.js" - }, - "dependencies": { - "request": "^2.88.0", - "request-promise": "^4.2.2" - } -} diff --git a/scripts/translations/unused.js b/scripts/translations/unused.js deleted file mode 100644 index 7a4ec0e9..00000000 --- a/scripts/translations/unused.js +++ /dev/null @@ -1,63 +0,0 @@ -const fs = require('fs'); -const path = require('path'); - -const SRC_DIR = '../../client/src/' -const LOCALES_DIR = '../../client/src/__locales'; -const BASE_FILE = path.join(LOCALES_DIR, 'en.json'); - -// Strings that may be found by the algorithm, -// but in fact they are used. -const KNOWN_USED_STRINGS = { - 'blocking_mode_refused': true, - 'blocking_mode_nxdomain': true, - 'blocking_mode_custom_ip': true, -} - -function traverseDir(dir, callback) { - fs.readdirSync(dir).forEach(file => { - let fullPath = path.join(dir, file); - if (fs.lstatSync(fullPath).isDirectory()) { - traverseDir(fullPath, callback); - } else { - callback(fullPath); - } - }); -} - -const contains = (key, files) => { - for (let file of files) { - if (file.includes(key)) { - return true; - } - } - - return false; -} - -const main = () => { - const baseLanguage = require(BASE_FILE); - const files = []; - - traverseDir(SRC_DIR, (path) => { - const canContain = (path.endsWith('.js') || path.endsWith('.json')) && - !path.includes(LOCALES_DIR); - - if (canContain) { - files.push(fs.readFileSync(path).toString()); - } - }); - - const unused = []; - for (let key in baseLanguage) { - if (!contains(key, files) && !KNOWN_USED_STRINGS[key]) { - unused.push(key); - } - } - - console.log('Unused keys:'); - for (let key of unused) { - console.log(key); - } -} - -main(); diff --git a/scripts/translations/upload.js b/scripts/translations/upload.js deleted file mode 100644 index 702b251b..00000000 --- a/scripts/translations/upload.js +++ /dev/null @@ -1,47 +0,0 @@ -const path = require('path'); -const fs = require('fs'); -const request = require('request-promise'); -const twoskyConfig = require('../../.twosky.json')[0]; - -const { project_id: TWOSKY_PROJECT_ID, base_locale: DEFAULT_LANGUAGE } = twoskyConfig; -const LOCALES_DIR = '../../client/src/__locales'; -const BASE_FILE = 'en.json'; -const TWOSKY_URI = process.env.TWOSKY_URI; - -/** - * Prepare post params - */ -const getRequestData = (url, projectId) => { - const language = process.env.UPLOAD_LANGUAGE || DEFAULT_LANGUAGE; - const formData = { - format: 'json', - language: language, - filename: BASE_FILE, - project: projectId, - file: fs.createReadStream(path.resolve(LOCALES_DIR, `${language}.json`)), - }; - - console.log(`uploading ${language}`); - - return { - url: `${url}/upload`, - formData - }; -}; - -/** - * Make request to twosky to upload new json - */ -const upload = () => { - if (!TWOSKY_URI) { - console.error('No credentials'); - return; - } - - const { url, formData } = getRequestData(TWOSKY_URI, TWOSKY_PROJECT_ID); - request - .post({ url, formData }) - .catch(err => console.log(err)); -}; - -upload();