mirrors
/
AdGuardHome
mirror of https://github.com/AdguardTeam/AdGuardHome.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into 4299-querylog-stats-api

This commit is contained in:
Stanislav Chzhen 2023-03-22 13:53:09 +03:00
parent 48b62b0f18 306c1983a2
commit 5b706b7997
30 changed files with 865 additions and 1251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -31,6 +31,8 @@ NOTE: Add new changes BELOW THIS COMMENT.
- Two new HTTP APIs, `PUT /control/querylog/config/update` and `GET
control/querylog/config`, which can be used to set and receive the statistics
configuration. See openapi/openapi.yaml for the full description.
- The ability to set custom IP for EDNS Client Subnet by using the DNS-server
configuration section on the DNS settings page in the UI ([#1472]).
- The ability to manage safesearch for each service by using the new
`safe_search` field ([#1163]).
@ -109,6 +111,7 @@ In this release, the schema version has changed from 17 to 20.
([#5584]).
[#1163]: https://github.com/AdguardTeam/AdGuardHome/issues/1163
[#1472]: https://github.com/AdguardTeam/AdGuardHome/issues/1472
[#5567]: https://github.com/AdguardTeam/AdGuardHome/issues/5567
[#5584]: https://github.com/AdguardTeam/AdGuardHome/issues/5584

5
client/src/__locales/en.json
View File

@ -290,6 +290,8 @@
"rate_limit": "Rate limit",
"edns_enable": "Enable EDNS client subnet",
"edns_cs_desc": "Add the EDNS Client Subnet option (ECS) to upstream requests and log the values sent by the clients in the query log.",
"edns_use_custom_ip": "Use custom IP for EDNS",
"edns_use_custom_ip_desc": "Allow to use custom IP for EDNS",
"rate_limit_desc": "The number of requests per second allowed per client. Setting it to 0 means no limit.",
"blocking_ipv4_desc": "IP address to be returned for a blocked A request",
"blocking_ipv6_desc": "IP address to be returned for a blocked AAAA request",
@ -646,5 +648,6 @@
"anonymizer_notification": "<0>Note:</0> IP anonymization is enabled. You can disable it in <1>General settings</1>.",
"confirm_dns_cache_clear": "Are you sure you want to clear DNS cache?",
"cache_cleared": "DNS cache successfully cleared",
"clear_cache": "Clear cache"
"clear_cache": "Clear cache",
"make_static": "Make static"
}

6
client/src/components/Dashboard/BlockedDomains.js
View File

@ -29,8 +29,11 @@ const BlockedDomains = ({
blockedFiltering,
replacedSafebrowsing,
replacedParental,
replacedSafesearch,
}) => {
const totalBlocked = blockedFiltering + replacedSafebrowsing + replacedParental;
const totalBlocked = (
blockedFiltering + replacedSafebrowsing + replacedParental + replacedSafesearch
);
return (
<Card
@ -71,6 +74,7 @@ BlockedDomains.propTypes = {
topBlockedDomains: PropTypes.array.isRequired,
blockedFiltering: PropTypes.number.isRequired,
replacedSafebrowsing: PropTypes.number.isRequired,
replacedSafesearch: PropTypes.number.isRequired,
replacedParental: PropTypes.number.isRequired,
refreshButton: PropTypes.node.isRequired,
subtitle: PropTypes.string.isRequired,

1
client/src/components/Dashboard/index.js
View File

@ -151,6 +151,7 @@ const Dashboard = ({
topBlockedDomains={stats.topBlockedDomains}
blockedFiltering={stats.numBlockedFiltering}
replacedSafebrowsing={stats.numReplacedSafebrowsing}
replacedSafesearch={stats.numReplacedSafesearch}
replacedParental={stats.numReplacedParental}
refreshButton={refreshButton}
/>

37
client/src/components/Settings/Dhcp/Leases.js
View File

@ -1,9 +1,11 @@
import React, { Component } from 'react';
import { connect } from 'react-redux';
import PropTypes from 'prop-types';
import ReactTable from 'react-table';
import { Trans, withTranslation } from 'react-i18next';
import { LEASES_TABLE_DEFAULT_PAGE_SIZE } from '../../../helpers/constants';
import { sortIp } from '../../../helpers/helpers';
import { toggleLeaseModal } from '../../../actions';
class Leases extends Component {
cellWrap = ({ value }) => (
@ -14,6 +16,30 @@ class Leases extends Component {
</div>
);
convertToStatic = (data) => () => {
const { dispatch } = this.props;
dispatch(toggleLeaseModal(data));
}
makeStatic = ({ row }) => {
const { t, disabledLeasesButton } = this.props;
return (
<div className="logs__row logs__row--center">
<button
type="button"
className="btn btn-icon btn-icon--green btn-outline-secondary btn-sm"
title={t('make_static')}
onClick={this.convertToStatic(row)}
disabled={disabledLeasesButton}
>
<svg className="icons icon12">
<use xlinkHref="#plus" />
</svg>
</button>
</div>
);
}
render() {
const { leases, t } = this.props;
return (
@ -23,7 +49,7 @@ class Leases extends Component {
{
Header: 'MAC',
accessor: 'mac',
minWidth: 160,
minWidth: 180,
Cell: this.cellWrap,
}, {
Header: 'IP',
@ -39,8 +65,11 @@ class Leases extends Component {
}, {
Header: <Trans>dhcp_table_expires</Trans>,
accessor: 'expires',
minWidth: 130,
minWidth: 220,
Cell: this.cellWrap,
}, {
Header: <Trans>actions_table_header</Trans>,
Cell: this.makeStatic,
},
]}
pageSize={LEASES_TABLE_DEFAULT_PAGE_SIZE}
@ -57,6 +86,8 @@ class Leases extends Component {
Leases.propTypes = {
leases: PropTypes.array,
t: PropTypes.func,
dispatch: PropTypes.func,
disabledLeasesButton: PropTypes.bool,
};
export default withTranslation()(Leases);
export default withTranslation()(connect(() => ({}), (dispatch) => ({ dispatch }))(Leases));

5
client/src/components/Settings/Dhcp/StaticLeases/Form.js
View File

@ -2,7 +2,7 @@ import React from 'react';
import PropTypes from 'prop-types';
import { Field, reduxForm } from 'redux-form';
import { Trans, useTranslation } from 'react-i18next';
import { useDispatch } from 'react-redux';
import { useDispatch, useSelector, shallowEqual } from 'react-redux';
import { renderInputField, normalizeMac } from '../../../../helpers/form';
import {
@ -25,6 +25,7 @@ const Form = ({
}) => {
const { t } = useTranslation();
const dispatch = useDispatch();
const dynamicLease = useSelector((store) => store.dhcp.leaseModalConfig, shallowEqual);
const onClick = () => {
reset();
@ -87,7 +88,7 @@ const Form = ({
<button
type="submit"
className="btn btn-success btn-standard"
disabled={submitting || pristine || processingAdding}
disabled={submitting || processingAdding || (pristine && !dynamicLease)}
>
<Trans>save_btn</Trans>
</button>

11
client/src/components/Settings/Dhcp/StaticLeases/Modal.js
View File

@ -2,7 +2,7 @@ import React from 'react';
import PropTypes from 'prop-types';
import { Trans, withTranslation } from 'react-i18next';
import ReactModal from 'react-modal';
import { useDispatch } from 'react-redux';
import { shallowEqual, useDispatch, useSelector } from 'react-redux';
import Form from './Form';
import { toggleLeaseModal } from '../../../../actions';
@ -18,6 +18,9 @@ const Modal = ({
const dispatch = useDispatch();
const toggleModal = () => dispatch(toggleLeaseModal());
const leaseInitialData = useSelector(
(state) => state.dhcp.leaseModalConfig, shallowEqual,
) || {};
return (
<ReactModal
@ -37,9 +40,9 @@ const Modal = ({
</div>
<Form
initialValues={{
mac: '',
ip: '',
hostname: '',
mac: leaseInitialData.mac ?? '',
ip: leaseInitialData.ip ?? '',
hostname: leaseInitialData.hostname ?? '',
cidr,
rangeStart,
rangeEnd,

2
client/src/components/Settings/Dhcp/StaticLeases/index.js
View File

@ -54,7 +54,7 @@ const StaticLeases = ({
{
Header: 'MAC',
accessor: 'mac',
minWidth: 160,
minWidth: 180,
Cell: cellWrap,
},
{

6
client/src/components/Settings/Dhcp/index.js
View File

@ -188,8 +188,8 @@ const Dhcp = () => {
const inputtedIPv4values = dhcp?.values?.v4?.gateway_ip && dhcp?.values?.v4?.subnet_mask;
const isEmptyConfig = !Object.values(dhcp?.values?.v4 ?? {}).some(Boolean);
const disabledLeasesButton = dhcp?.syncErrors || interfaces?.syncErrors
|| !isInterfaceIncludesIpv4 || isEmptyConfig || processingConfig || !inputtedIPv4values;
const disabledLeasesButton = Boolean(dhcp?.syncErrors || interfaces?.syncErrors
|| !isInterfaceIncludesIpv4 || isEmptyConfig || processingConfig || !inputtedIPv4values);
const cidr = inputtedIPv4values ? `${dhcp?.values?.v4?.gateway_ip}/${subnetMaskToBitMask(dhcp?.values?.v4?.subnet_mask)}` : '';
return <>
@ -260,7 +260,7 @@ const Dhcp = () => {
>
<div className="row">
<div className="col">
<Leases leases={leases} />
<Leases leases={leases} disabledLeasesButton={disabledLeasesButton}/>
</div>
</div>
</Card>}

41
client/src/components/Settings/Dns/Config/Form.js
View File

@ -13,15 +13,11 @@ import {
validateIpv4,
validateIpv6,
validateRequiredValue,
validateIp,
} from '../../../../helpers/validators';
import { BLOCKING_MODES, FORM_NAME, UINT32_RANGE } from '../../../../helpers/constants';
const checkboxes = [
{
name: 'edns_cs_enabled',
placeholder: 'edns_enable',
subtitle: 'edns_cs_desc',
},
{
name: 'dnssec_enabled',
placeholder: 'dnssec_enable',
@ -66,6 +62,8 @@ const Form = ({
const { t } = useTranslation();
const {
blocking_mode,
edns_cs_enabled,
edns_cs_use_custom,
} = useSelector((state) => state.form[FORM_NAME.BLOCKING_MODE].values ?? {}, shallowEqual);
return <form onSubmit={handleSubmit}>
@ -92,6 +90,39 @@ const Form = ({
/>
</div>
</div>
<div className="col-12">
<div className="form__group form__group--settings">
<Field
name="edns_cs_enabled"
type="checkbox"
component={CheckboxField}
placeholder={t('edns_enable')}
disabled={processing}
subtitle={t('edns_cs_desc')}
/>
</div>
</div>
<div className="col-12 form__group form__group--inner">
<div className="form__group ">
<Field
name="edns_cs_use_custom"
type="checkbox"
component={CheckboxField}
placeholder={t('edns_use_custom_ip')}
disabled={processing || !edns_cs_enabled}
subtitle={t('edns_use_custom_ip_desc')}
/>
</div>
{edns_cs_use_custom && (<Field
name="edns_cs_custom_ip"
component={renderInputField}
className="form-control"
placeholder={t('form_enter_ip')}
validate={[validateIp, validateRequiredValue]}
/>)}
</div>
{checkboxes.map(({ name, placeholder, subtitle }) => <div className="col-12" key={name}>
<div className="form__group form__group--settings">
<Field

4
client/src/components/Settings/Dns/Config/index.js
View File

@ -14,6 +14,8 @@ const Config = () => {
blocking_ipv4,
blocking_ipv6,
edns_cs_enabled,
edns_cs_use_custom,
edns_cs_custom_ip,
dnssec_enabled,
disable_ipv6,
processingSetConfig,
@ -39,6 +41,8 @@ const Config = () => {
edns_cs_enabled,
disable_ipv6,
dnssec_enabled,
edns_cs_use_custom,
edns_cs_custom_ip,
}}
onSubmit={handleFormSubmit}
processing={processingSetConfig}

4
client/src/reducers/dhcp.js
View File

@ -124,10 +124,11 @@ const dhcp = handleActions(
staticLeases: [],
}),
[actions.toggleLeaseModal]: (state) => {
[actions.toggleLeaseModal]: (state, { payload }) => {
const newState = {
...state,
isModalOpen: !state.isModalOpen,
leaseModalConfig: payload,
};
return newState;
},
@ -200,6 +201,7 @@ const dhcp = handleActions(
leases: [],
staticLeases: [],
isModalOpen: false,
leaseModalConfig: undefined,
dhcp_available: false,
},
);

15
internal/dnsforward/config.go
View File

@ -200,7 +200,7 @@ type FilteringConfig struct {
// EDNSClientSubnet is the settings list for EDNS Client Subnet.
type EDNSClientSubnet struct {
// CustomIP for EDNS Client Subnet.
CustomIP string `yaml:"custom_ip"`
CustomIP netip.Addr `yaml:"custom_ip"`
// Enabled defines if EDNS Client Subnet is enabled.
Enabled bool `yaml:"enabled"`
@ -340,15 +340,8 @@ func (s *Server) createProxyConfig() (conf proxy.Config, err error) {
}
if srvConf.EDNSClientSubnet.UseCustom {
// TODO(s.chzhen): Add wrapper around netip.Addr.
var ip net.IP
ip, err = netutil.ParseIP(srvConf.EDNSClientSubnet.CustomIP)
if err != nil {
return conf, fmt.Errorf("edns: %w", err)
}
// TODO(s.chzhen): Use netip.Addr instead of net.IP inside dnsproxy.
conf.EDNSAddr = ip
conf.EDNSAddr = net.IP(srvConf.EDNSClientSubnet.CustomIP.AsSlice())
}
if srvConf.CacheSize != 0 {
@ -377,7 +370,7 @@ func (s *Server) createProxyConfig() (conf proxy.Config, err error) {
err = s.prepareTLS(&conf)
if err != nil {
return conf, fmt.Errorf("validating tls: %w", err)
return proxy.Config{}, fmt.Errorf("validating tls: %w", err)
}
if c := srvConf.DNSCryptConfig; c.Enabled {
@ -388,7 +381,7 @@ func (s *Server) createProxyConfig() (conf proxy.Config, err error) {
}
if conf.UpstreamConfig == nil || len(conf.UpstreamConfig.Upstreams) == 0 {
return conf, errors.Error("no default upstream servers configured")
return proxy.Config{}, errors.Error("no default upstream servers configured")
}
return conf, nil

164
internal/dnsforward/http.go
View File

@ -23,26 +23,78 @@ import (
)
// jsonDNSConfig is the JSON representation of the DNS server configuration.
//
// TODO(s.chzhen): Split it into smaller pieces. Use aghalg.NullBool instead
// of *bool.
type jsonDNSConfig struct {
Upstreams *[]string `json:"upstream_dns"`
UpstreamsFile *string `json:"upstream_dns_file"`
Bootstraps *[]string `json:"bootstrap_dns"`
ProtectionEnabled *bool `json:"protection_enabled"`
RateLimit *uint32 `json:"ratelimit"`
BlockingMode *BlockingMode `json:"blocking_mode"`
EDNSCSEnabled *bool `json:"edns_cs_enabled"`
DNSSECEnabled *bool `json:"dnssec_enabled"`
DisableIPv6 *bool `json:"disable_ipv6"`
UpstreamMode *string `json:"upstream_mode"`
CacheSize *uint32 `json:"cache_size"`
CacheMinTTL *uint32 `json:"cache_ttl_min"`
CacheMaxTTL *uint32 `json:"cache_ttl_max"`
CacheOptimistic *bool `json:"cache_optimistic"`
ResolveClients *bool `json:"resolve_clients"`
UsePrivateRDNS *bool `json:"use_private_ptr_resolvers"`
LocalPTRUpstreams *[]string `json:"local_ptr_upstreams"`
BlockingIPv4 net.IP `json:"blocking_ipv4"`
BlockingIPv6 net.IP `json:"blocking_ipv6"`
// Upstreams is the list of upstream DNS servers.
Upstreams *[]string `json:"upstream_dns"`
// UpstreamsFile is the file containing upstream DNS servers.
UpstreamsFile *string `json:"upstream_dns_file"`
// Bootstraps is the list of DNS servers resolving IP addresses of the
// upstream DoH/DoT resolvers.
Bootstraps *[]string `json:"bootstrap_dns"`
// ProtectionEnabled defines if protection is enabled.
ProtectionEnabled *bool `json:"protection_enabled"`
// RateLimit is the number of requests per second allowed per client.
RateLimit *uint32 `json:"ratelimit"`
// BlockingMode defines the way blocked responses are constructed.
BlockingMode *BlockingMode `json:"blocking_mode"`
// EDNSCSEnabled defines if EDNS Client Subnet is enabled.
EDNSCSEnabled *bool `json:"edns_cs_enabled"`
// EDNSCSUseCustom defines if EDNSCSCustomIP should be used.
EDNSCSUseCustom *bool `json:"edns_cs_use_custom"`
// DNSSECEnabled defines if DNSSEC is enabled.
DNSSECEnabled *bool `json:"dnssec_enabled"`
// DisableIPv6 defines if IPv6 addresses should be dropped.
DisableIPv6 *bool `json:"disable_ipv6"`
// UpstreamMode defines the way DNS requests are constructed.
UpstreamMode *string `json:"upstream_mode"`
// CacheSize in bytes.
CacheSize *uint32 `json:"cache_size"`
// CacheMinTTL is custom minimum TTL for cached DNS responses.
CacheMinTTL *uint32 `json:"cache_ttl_min"`
// CacheMaxTTL is custom maximum TTL for cached DNS responses.
CacheMaxTTL *uint32 `json:"cache_ttl_max"`
// CacheOptimistic defines if expired entries should be served.
CacheOptimistic *bool `json:"cache_optimistic"`
// ResolveClients defines if clients IPs should be resolved into hostnames.
ResolveClients *bool `json:"resolve_clients"`
// UsePrivateRDNS defines if privates DNS resolvers should be used.
UsePrivateRDNS *bool `json:"use_private_ptr_resolvers"`
// LocalPTRUpstreams is the list of local private DNS resolvers.
LocalPTRUpstreams *[]string `json:"local_ptr_upstreams"`
// BlockingIPv4 is custom IPv4 address for blocked A requests.
BlockingIPv4 net.IP `json:"blocking_ipv4"`
// BlockingIPv6 is custom IPv6 address for blocked AAAA requests.
BlockingIPv6 net.IP `json:"blocking_ipv6"`
// EDNSCSCustomIP is custom IP for EDNS Client Subnet.
EDNSCSCustomIP netip.Addr `json:"edns_cs_custom_ip"`
// DefaultLocalPTRUpstreams is used to pass the addresses from
// systemResolvers to the front-end. It's not a pointer to the slice since
// there is no need to omit it while decoding from JSON.
DefaultLocalPTRUpstreams []string `json:"default_local_ptr_upstreams,omitempty"`
}
func (s *Server) getDNSConfig() (c *jsonDNSConfig) {
@ -57,7 +109,11 @@ func (s *Server) getDNSConfig() (c *jsonDNSConfig) {
blockingIPv4 := s.conf.BlockingIPv4
blockingIPv6 := s.conf.BlockingIPv6
ratelimit := s.conf.Ratelimit
customIP := s.conf.EDNSClientSubnet.CustomIP
enableEDNSClientSubnet := s.conf.EDNSClientSubnet.Enabled
useCustom := s.conf.EDNSClientSubnet.UseCustom
enableDNSSEC := s.conf.EnableDNSSEC
aaaaDisabled := s.conf.AAAADisabled
cacheSize := s.conf.CacheSize
@ -74,46 +130,40 @@ func (s *Server) getDNSConfig() (c *jsonDNSConfig) {
upstreamMode = "parallel"
}
return &jsonDNSConfig{
Upstreams: &upstreams,
UpstreamsFile: &upstreamFile,
Bootstraps: &bootstraps,
ProtectionEnabled: &protectionEnabled,
BlockingMode: &blockingMode,
BlockingIPv4: blockingIPv4,
BlockingIPv6: blockingIPv6,
RateLimit: &ratelimit,
EDNSCSEnabled: &enableEDNSClientSubnet,
DNSSECEnabled: &enableDNSSEC,
DisableIPv6: &aaaaDisabled,
CacheSize: &cacheSize,
CacheMinTTL: &cacheMinTTL,
CacheMaxTTL: &cacheMaxTTL,
CacheOptimistic: &cacheOptimistic,
UpstreamMode: &upstreamMode,
ResolveClients: &resolveClients,
UsePrivateRDNS: &usePrivateRDNS,
LocalPTRUpstreams: &localPTRUpstreams,
}
}
func (s *Server) handleGetConfig(w http.ResponseWriter, r *http.Request) {
defLocalPTRUps, err := s.filterOurDNSAddrs(s.sysResolvers.Get())
if err != nil {
log.Debug("getting dns configuration: %s", err)
}
resp := struct {
jsonDNSConfig
// DefautLocalPTRUpstreams is used to pass the addresses from
// systemResolvers to the front-end. It's not a pointer to the slice
// since there is no need to omit it while decoding from JSON.
DefautLocalPTRUpstreams []string `json:"default_local_ptr_upstreams,omitempty"`
}{
jsonDNSConfig: *s.getDNSConfig(),
DefautLocalPTRUpstreams: defLocalPTRUps,
return &jsonDNSConfig{
Upstreams: &upstreams,
UpstreamsFile: &upstreamFile,
Bootstraps: &bootstraps,
ProtectionEnabled: &protectionEnabled,
BlockingMode: &blockingMode,
BlockingIPv4: blockingIPv4,
BlockingIPv6: blockingIPv6,
RateLimit: &ratelimit,
EDNSCSCustomIP: customIP,
EDNSCSEnabled: &enableEDNSClientSubnet,
EDNSCSUseCustom: &useCustom,
DNSSECEnabled: &enableDNSSEC,
DisableIPv6: &aaaaDisabled,
CacheSize: &cacheSize,
CacheMinTTL: &cacheMinTTL,
CacheMaxTTL: &cacheMaxTTL,
CacheOptimistic: &cacheOptimistic,
UpstreamMode: &upstreamMode,
ResolveClients: &resolveClients,
UsePrivateRDNS: &usePrivateRDNS,
LocalPTRUpstreams: &localPTRUpstreams,
DefaultLocalPTRUpstreams: defLocalPTRUps,
}
}
// handleGetConfig handles requests to the GET /control/dns_info endpoint.
func (s *Server) handleGetConfig(w http.ResponseWriter, r *http.Request) {
resp := s.getDNSConfig()
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
@ -204,6 +254,7 @@ func (req *jsonDNSConfig) checkCacheTTL() bool {
return min <= max
}
// handleSetConfig handles requests to the POST /control/dns_config endpoint.
func (s *Server) handleSetConfig(w http.ResponseWriter, r *http.Request) {
req := &jsonDNSConfig{}
err := json.NewDecoder(r.Body).Decode(req)
@ -231,8 +282,8 @@ func (s *Server) handleSetConfig(w http.ResponseWriter, r *http.Request) {
}
}
// setConfigRestartable sets the server parameters. shouldRestart is true if
// the server should be restarted to apply changes.
// setConfig sets the server parameters. shouldRestart is true if the server
// should be restarted to apply changes.
func (s *Server) setConfig(dc *jsonDNSConfig) (shouldRestart bool) {
s.serverLock.Lock()
defer s.serverLock.Unlock()
@ -250,6 +301,10 @@ func (s *Server) setConfig(dc *jsonDNSConfig) (shouldRestart bool) {
s.conf.FastestAddr = *dc.UpstreamMode == "fastest_addr"
}
if dc.EDNSCSUseCustom != nil && *dc.EDNSCSUseCustom {
s.conf.EDNSClientSubnet.CustomIP = dc.EDNSCSCustomIP
}
setIfNotNil(&s.conf.ProtectionEnabled, dc.ProtectionEnabled)
setIfNotNil(&s.conf.EnableDNSSEC, dc.DNSSECEnabled)
setIfNotNil(&s.conf.AAAADisabled, dc.DisableIPv6)
@ -281,6 +336,7 @@ func (s *Server) setConfigRestartable(dc *jsonDNSConfig) (shouldRestart bool) {
setIfNotNil(&s.conf.UpstreamDNSFileName, dc.UpstreamsFile),
setIfNotNil(&s.conf.BootstrapDNS, dc.Bootstraps),
setIfNotNil(&s.conf.EDNSClientSubnet.Enabled, dc.EDNSCSEnabled),
setIfNotNil(&s.conf.EDNSClientSubnet.UseCustom, dc.EDNSCSUseCustom),
setIfNotNil(&s.conf.CacheSize, dc.CacheSize),
setIfNotNil(&s.conf.CacheMinTTL, dc.CacheMinTTL),
setIfNotNil(&s.conf.CacheMaxTTL, dc.CacheMaxTTL),

16
internal/dnsforward/http_test.go
View File

@ -181,6 +181,12 @@ func TestDNSForwardHTTP_handleSetConfig(t *testing.T) {
}, {
name: "edns_cs_enabled",
wantSet: "",
}, {
name: "edns_cs_use_custom",
wantSet: "",
}, {
name: "edns_cs_use_custom_bad_ip",
wantSet: "decoding request: ParseAddr(\"bad.ip\"): unexpected character (at \"bad.ip\")",
}, {
name: "dnssec_enabled",
wantSet: "",
@ -222,16 +228,20 @@ func TestDNSForwardHTTP_handleSetConfig(t *testing.T) {
Req json.RawMessage `json:"req"`
Want json.RawMessage `json:"want"`
}
loadTestData(t, t.Name()+jsonExt, &data)
testData := t.Name() + jsonExt
loadTestData(t, testData, &data)
for _, tc := range testCases {
// NOTE: Do not use require.Contains, because the size of the data
// prevents it from printing a meaningful error message.
caseData, ok := data[tc.name]
require.True(t, ok)
require.Truef(t, ok, "%q does not contain test data for test case %s", testData, tc.name)
t.Run(tc.name, func(t *testing.T) {
t.Cleanup(func() {
s.conf = defaultConf
s.conf.FilteringConfig.EDNSClientSubnet.Enabled = false
s.conf.FilteringConfig.EDNSClientSubnet = &EDNSClientSubnet{}
})
rBody := io.NopCloser(bytes.NewReader(caseData.Req))

12
internal/dnsforward/testdata/TestDNSForwardHTTP_handleGetConfig.json vendored
View File

@ -26,7 +26,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
},
"fastest_addr": {
"upstream_dns": [
@ -55,7 +57,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
},
"parallel": {
"upstream_dns": [
@ -84,6 +88,8 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
}

144
internal/dnsforward/testdata/TestDNSForwardHTTP_handleSetConfig.json vendored
View File

@ -33,7 +33,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"bootstraps": {
@ -66,7 +68,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"blocking_mode_good": {
@ -100,7 +104,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"blocking_mode_bad": {
@ -134,7 +140,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"ratelimit": {
@ -168,7 +176,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"edns_cs_enabled": {
@ -202,7 +212,85 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"edns_cs_use_custom": {
"req": {
"edns_cs_enabled": true,
"edns_cs_use_custom": true,
"edns_cs_custom_ip": "1.2.3.4"
},
"want": {
"upstream_dns": [
"8.8.8.8:53",
"8.8.4.4:53"
],
"upstream_dns_file": "",
"bootstrap_dns": [
"9.9.9.10",
"149.112.112.10",
"2620:fe::10",
"2620:fe::fe:10"
],
"protection_enabled": true,
"ratelimit": 0,
"blocking_mode": "default",
"blocking_ipv4": "",
"blocking_ipv6": "",
"edns_cs_enabled": true,
"dnssec_enabled": false,
"disable_ipv6": false,
"upstream_mode": "",
"cache_size": 0,
"cache_ttl_min": 0,
"cache_ttl_max": 0,
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": [],
"edns_cs_use_custom": true,
"edns_cs_custom_ip": "1.2.3.4"
}
},
"edns_cs_use_custom_bad_ip": {
"req": {
"edns_cs_enabled": true,
"edns_cs_use_custom": true,
"edns_cs_custom_ip": "bad.ip"
},
"want": {
"upstream_dns": [
"8.8.8.8:53",
"8.8.4.4:53"
],
"upstream_dns_file": "",
"bootstrap_dns": [
"9.9.9.10",
"149.112.112.10",
"2620:fe::10",
"2620:fe::fe:10"
],
"protection_enabled": true,
"ratelimit": 0,
"blocking_mode": "default",
"blocking_ipv4": "",
"blocking_ipv6": "",
"edns_cs_enabled": false,
"dnssec_enabled": false,
"disable_ipv6": false,
"upstream_mode": "",
"cache_size": 0,
"cache_ttl_min": 0,
"cache_ttl_max": 0,
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"dnssec_enabled": {
@ -236,7 +324,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"cache_size": {
@ -270,7 +360,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"upstream_mode_parallel": {
@ -304,7 +396,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"upstream_mode_fastest_addr": {
@ -338,7 +432,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"upstream_dns_bad": {
@ -374,7 +470,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"bootstraps_bad": {
@ -410,7 +508,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"cache_bad_ttl": {
@ -445,7 +545,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"upstream_mode_bad": {
@ -479,7 +581,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"local_ptr_upstreams_good": {
@ -517,7 +621,9 @@
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": [
"123.123.123.123"
]
],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"local_ptr_upstreams_bad": {
@ -554,7 +660,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
},
"local_ptr_upstreams_null": {
@ -588,7 +696,9 @@
"cache_optimistic": false,
"resolve_clients": false,
"use_private_ptr_resolvers": false,
"local_ptr_upstreams": []
"local_ptr_upstreams": [],
"edns_cs_use_custom": false,
"edns_cs_custom_ip": ""
}
}
}

2
internal/home/config.go
View File

@ -284,7 +284,7 @@ var config = &configuration{
CacheSize: 4 * 1024 * 1024,
EDNSClientSubnet: &dnsforward.EDNSClientSubnet{
CustomIP: "",
CustomIP: netip.Addr{},
Enabled: false,
UseCustom: false,
},

12
openapi/CHANGELOG.md
View File

@ -4,6 +4,18 @@
## v0.108.0: API changes
## v0.107.27: API changes
### The new optional fields `"edns_cs_use_custom"` and `"edns_cs_custom_ip"` in `DNSConfig`
* The new optional fields `"edns_cs_use_custom"` and `"edns_cs_custom_ip"` in
`POST /control/dns_config` method makes AdGuard Home use or not use the
custom IP for EDNS Client Subnet.
* The new optional fields `"edns_cs_use_custom"` and `"edns_cs_custom_ip"` in
`GET /control/dns_info` method are set if AdGuard Home uses custom IP for
EDNS Client Subnet.
## v0.107.27: API changes

8
openapi/openapi.yaml
View File

@ -1332,7 +1332,7 @@
'example': 'en'
'DNSConfig':
'type': 'object'
'description': 'Query log configuration'
'description': 'DNS server configuration'
'properties':
'bootstrap_dns':
'type': 'array'
@ -1358,8 +1358,6 @@
'type': 'string'
'protection_enabled':
'type': 'boolean'
'dhcp_available':
'type': 'boolean'
'ratelimit':
'type': 'integer'
'blocking_mode':
@ -1376,6 +1374,10 @@
'type': 'string'
'edns_cs_enabled':
'type': 'boolean'
'edns_cs_use_custom':
'type': 'boolean'
'edns_cs_custom_ip':
'type': 'string'
'disable_ipv6':
'type': 'boolean'
'dnssec_enabled':

24
scripts/README.md
View File

@ -178,22 +178,28 @@ manifest file templates, and helper scripts.
### Usage
* `npm install`: install dependencies. Run this first.
* `npm run locales:download`: download and save all translations.
* `npm run locales:upload`: upload the base `en` locale.
* `npm run locales:summary`: show the current locales summary.
* `npm run locales:unused`: show the list of unused strings.
* `go run main.go help`: print usage.
* `go run main.go download [-n <count>]`: download and save all translations.
`n` is optional flag where count is a number of concurrent downloads.
* `go run main.go upload`: upload the base `en` locale.
* `go run main.go summary`: show the current locales summary.
* `go run main.go unused`: show the list of unused strings.
After the download you'll find the output locales in the `client/src/__locales/`
directory.
Optional environment:
* `SLEEP_TIME`: set the sleep time between downloads for `locales:download`,
in milliseconds. The default is 250 ms.
* `UPLOAD_LANGUAGE`: set an alternative language for `upload`.
* `UPLOAD_LANGUAGE`: set an alternative language for `locales:upload` to
upload.
* `TWOSKY_URI`: set an alternative URL for `download` or `upload`.
* `TWOSKY_PROJECT_ID`: set an alternative project ID for `download` or
`upload`.

1
scripts/make/go-lint.sh
View File

@ -182,6 +182,7 @@ run_linter gocyclo --over 10\
./internal/version/\
./scripts/blocked-services/\
./scripts/vetted-filters/\
./scripts/translations/\
./main.go\
;

1
scripts/translations/.gitignore vendored
View File

@ -1 +0,0 @@
node_modules

41
scripts/translations/count.js
View File

@ -1,41 +0,0 @@
const path = require('path');
const twoskyConfig = require('../../.twosky.json')[0];
const {languages} = twoskyConfig;
const LOCALES_DIR = '../../client/src/__locales';
const LOCALES_LIST = Object.keys(languages);
const BASE_FILE = 'en.json';
const main = () => {
const pathToBaseFile = path.join(LOCALES_DIR, BASE_FILE);
const baseLanguageJson = require(pathToBaseFile);
const summary = {};
LOCALES_LIST.forEach((locale) => {
const pathToFile = path.join(LOCALES_DIR, `${locale}.json`);
if (pathToFile === pathToBaseFile) {
return;
}
let total = 0;
let translated = 0;
const languageJson = require(pathToFile);
for (let key in baseLanguageJson) {
total += 1;
if (key in languageJson) {
translated += 1;
}
}
summary[locale] = Math.round(translated / total * 10000) / 100;
});
console.log('Translations summary:');
for (let key in summary) {
console.log(`${key}, translated ${summary[key]}%`);
}
}
main();

125
scripts/translations/download.js
View File

@ -1,125 +0,0 @@
// TODO(a.garipov): Rewrite this in Go; add better concurrency controls; add
// features for easier maintenance.
const fs = require('fs');
const path = require('path');
const requestPromise = require('request-promise');
const twoskyConfig = require('../../.twosky.json')[0];
const { project_id: TWOSKY_PROJECT_ID, languages } = twoskyConfig;
const LOCALES_DIR = '../../client/src/__locales';
const LOCALES_LIST = Object.keys(languages);
const BASE_FILE = 'en.json';
const TWOSKY_URI = process.env.TWOSKY_URI;
/**
* Prepare params to get translations from twosky
* @param {string} locale language shortcut
* @param {object} twosky config twosky
*/
const getRequestUrl = (locale, url, projectId) => {
return `${url}/download?format=json&language=${locale}&filename=${BASE_FILE}&project=${projectId}`;
};
/**
* Promise wrapper for writing in file
* @param {string} filename
* @param {any} body
*/
function writeInFile(filename, body) {
let normalizedBody = removeEmpty(JSON.parse(body));
return new Promise((resolve, reject) => {
if (typeof normalizedBody !== 'string') {
try {
normalizedBody = JSON.stringify(normalizedBody, null, 4) + '\n'; // eslint-disable-line
} catch (err) {
reject(err);
}
}
fs.writeFile(filename, normalizedBody, (err) => {
if (err) reject(err);
resolve('Ok');
});
});
}
/**
* Clear initial from empty value keys
* @param {object} initialObject
*/
function removeEmpty(initialObject) {
let processedObject = {};
Object.keys(initialObject).forEach(prop => {
if (initialObject[prop]) {
processedObject[prop] = initialObject[prop];
}
});
return processedObject;
}
/**
* Request twosky
* @param {string} url
* @param {string} locale
*/
const request = (url, locale) => (
requestPromise.get(url)
.then((res) => {
if (res.length) {
const pathToFile = path.join(LOCALES_DIR, `${locale}.json`);
return writeInFile(pathToFile, res);
}
return null;
})
.then((res) => {
let result = locale;
result += res ? ' - OK' : ' - Empty';
return result;
})
.catch((err) => {
console.log(err);
return `${locale} - Not OK`;
}));
/**
* Sleep.
* @param {number} ms
*/
const sleep = (ms) => new Promise((resolve) => {
setTimeout(resolve, ms);
});
/**
* Download locales
*/
const download = async () => {
const locales = LOCALES_LIST;
if (!TWOSKY_URI) {
console.error('No credentials');
return;
}
const requests = [];
for (let i = 0; i < locales.length; i++) {
const locale = locales[i];
const url = getRequestUrl(locale, TWOSKY_URI, TWOSKY_PROJECT_ID);
requests.push(request(url, locale));
// Don't request the Crowdin API too aggressively to prevent spurious
// 400 errors.
const sleepTime = process.env.SLEEP_TIME || 250;
await sleep(sleepTime);
}
Promise
.all(requests)
.then((res) => {
res.forEach(item => console.log(item));
})
.catch(err => console.log(err));
};
download();

464
scripts/translations/main.go Normal file
View File

@ -0,0 +1,464 @@
// translations downloads translations, uploads translations, prints summary
// for translations, prints unused strings.
package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghio"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
const (
twoskyConfFile = "./.twosky.json"
localesDir = "./client/src/__locales"
defaultBaseFile = "en.json"
defaultProjectID = "home"
srcDir = "./client/src"
twoskyURI = "https://twosky.int.agrd.dev/api/v1"
readLimit = 1 * 1024 * 1024
)
// langCode is a language code.
type langCode string
// languages is a map, where key is language code and value is display name.
type languages map[langCode]string
// textlabel is a text label of localization.
type textLabel string
// locales is a map, where key is text label and value is translation.
type locales map[textLabel]string
func main() {
if len(os.Args) == 1 {
usage("need a command")
}
if os.Args[1] == "help" {
usage("")
}
uriStr := os.Getenv("TWOSKY_URI")
if uriStr == "" {
uriStr = twoskyURI
}
uri, err := url.Parse(uriStr)
check(err)
projectID := os.Getenv("TWOSKY_PROJECT_ID")
if projectID == "" {
projectID = defaultProjectID
}
conf, err := readTwoskyConf()
check(err)
switch os.Args[1] {
case "summary":
err = summary(conf.Languages)
check(err)
case "download":
err = download(uri, projectID, conf.Languages)
check(err)
case "unused":
err = unused()
check(err)
case "upload":
err = upload(uri, projectID, conf.BaseLangcode)
check(err)
default:
usage("unknown command")
}
}
// check is a simple error-checking helper for scripts.
func check(err error) {
if err != nil {
panic(err)
}
}
// usage prints usage. If addStr is not empty print addStr and exit with code
// 1, otherwise exit with code 0.
func usage(addStr string) {
const usageStr = `Usage: go run main.go <command> [<args>]
Commands:
help
Print usage.
summary
Print summary.
download [-n <count>]
Download translations. count is a number of concurrent downloads.
unused
Print unused strings.
upload
Upload translations.`
if addStr != "" {
fmt.Printf("%s\n%s\n", addStr, usageStr)
os.Exit(1)
}
fmt.Println(usageStr)
os.Exit(0)
}
// twoskyConf is the configuration structure for localization.
type twoskyConf struct {
Languages languages `json:"languages"`
ProjectID string `json:"project_id"`
BaseLangcode langCode `json:"base_locale"`
LocalizableFiles []string `json:"localizable_files"`
}
// readTwoskyConf returns configuration.
func readTwoskyConf() (t twoskyConf, err error) {
b, err := os.ReadFile(twoskyConfFile)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return twoskyConf{}, err
}
var tsc []twoskyConf
err = json.Unmarshal(b, &tsc)
if err != nil {
err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
return twoskyConf{}, err
}
if len(tsc) == 0 {
err = fmt.Errorf("%q is empty", twoskyConfFile)
return twoskyConf{}, err
}
conf := tsc[0]
for _, lang := range conf.Languages {
if lang == "" {
return twoskyConf{}, errors.Error("language is empty")
}
}
return conf, nil
}
// readLocales reads file with name fn and returns a map, where key is text
// label and value is localization.
func readLocales(fn string) (loc locales, err error) {
b, err := os.ReadFile(fn)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil, err
}
loc = make(locales)
err = json.Unmarshal(b, &loc)
if err != nil {
err = fmt.Errorf("unmarshalling %q: %w", fn, err)
return nil, err
}
return loc, nil
}
// summary prints summary for translations.
func summary(langs languages) (err error) {
basePath := filepath.Join(localesDir, defaultBaseFile)
baseLoc, err := readLocales(basePath)
if err != nil {
return fmt.Errorf("summary: %w", err)
}
size := float64(len(baseLoc))
keys := maps.Keys(langs)
slices.Sort(keys)
for _, lang := range keys {
name := filepath.Join(localesDir, string(lang)+".json")
if name == basePath {
continue
}
var loc locales
loc, err = readLocales(name)
if err != nil {
return fmt.Errorf("summary: reading locales: %w", err)
}
f := float64(len(loc)) * 100 / size
fmt.Printf("%s\t %6.2f %%\n", lang, f)
}
return nil
}
// download and save all translations. uri is the base URL. projectID is the
// name of the project.
func download(uri *url.URL, projectID string, langs languages) (err error) {
var numWorker int
flagSet := flag.NewFlagSet("download", flag.ExitOnError)
flagSet.Usage = func() {
usage("download command error")
}
flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
err = flagSet.Parse(os.Args[2:])
if err != nil {
// Don't wrap the error since there is exit on error.
return err
}
if numWorker < 1 {
usage("count must be positive")
}
downloadURI := uri.JoinPath("download")
client := &http.Client{
Timeout: 10 * time.Second,
}
var wg sync.WaitGroup
uriCh := make(chan *url.URL, len(langs))
for i := 0; i < numWorker; i++ {
wg.Add(1)
go downloadWorker(&wg, client, uriCh)
}
for lang := range langs {
uri = translationURL(downloadURI, defaultBaseFile, projectID, lang)
uriCh <- uri
}
close(uriCh)
wg.Wait()
return nil
}
// downloadWorker downloads translations by received urls and saves them.
func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.URL) {
defer wg.Done()
for uri := range uriCh {
data, err := getTranslation(client, uri.String())
if err != nil {
log.Error("download worker: getting translation: %s", err)
continue
}
q := uri.Query()
code := q.Get("language")
name := filepath.Join(localesDir, code+".json")
err = os.WriteFile(name, data, 0o664)
if err != nil {
log.Error("download worker: writing file: %s", err)
continue
}
fmt.Println(name)
}
}
// getTranslation returns received translation data or error.
func getTranslation(client *http.Client, url string) (data []byte, err error) {
resp, err := client.Get(url)
if err != nil {
return nil, fmt.Errorf("requesting: %w", err)
}
defer log.OnCloserError(resp.Body, log.ERROR)
if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
return nil, err
}
limitReader, err := aghio.LimitReader(resp.Body, readLimit)
if err != nil {
err = fmt.Errorf("limit reading: %w", err)
return nil, err
}
data, err = io.ReadAll(limitReader)
if err != nil {
err = fmt.Errorf("reading all: %w", err)
return nil, err
}
return data, nil
}
// translationURL returns a new url.URL with provided query parameters.
func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
uri = &url.URL{}
*uri = *oldURL
q := uri.Query()
q.Set("format", "json")
q.Set("filename", baseFile)
q.Set("project", projectID)
q.Set("language", string(lang))
uri.RawQuery = q.Encode()
return uri
}
// unused prints unused text labels.
func unused() (err error) {
fileNames := []string{}
basePath := filepath.Join(localesDir, defaultBaseFile)
baseLoc, err := readLocales(basePath)
if err != nil {
return fmt.Errorf("unused: %w", err)
}
locDir := filepath.Clean(localesDir)
err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error {
if err != nil {
log.Info("accessing a path %q: %s", name, err)
return nil
}
if info.IsDir() {
return nil
}
if strings.HasPrefix(name, locDir) {
return nil
}
ext := filepath.Ext(name)
if ext == ".js" || ext == ".json" {
fileNames = append(fileNames, name)
}
return nil
})
if err != nil {
return fmt.Errorf("filepath walking %q: %w", srcDir, err)
}
err = removeUnused(fileNames, baseLoc)
return errors.Annotate(err, "removing unused: %w")
}
func removeUnused(fileNames []string, loc locales) (err error) {
knownUsed := []textLabel{
"blocking_mode_refused",
"blocking_mode_nxdomain",
"blocking_mode_custom_ip",
}
for _, v := range knownUsed {
delete(loc, v)
}
for _, fn := range fileNames {
var buf []byte
buf, err = os.ReadFile(fn)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
for k := range loc {
if bytes.Contains(buf, []byte(k)) {
delete(loc, k)
}
}
}
printUnused(loc)
return nil
}
// printUnused text labels to stdout.
func printUnused(loc locales) {
keys := maps.Keys(loc)
slices.Sort(keys)
for _, v := range keys {
fmt.Println(v)
}
}
// upload base translation. uri is the base URL. projectID is the name of the
// project. baseLang is the base language code.
func upload(uri *url.URL, projectID string, baseLang langCode) (err error) {
uploadURI := uri.JoinPath("upload")
lang := baseLang
langStr := os.Getenv("UPLOAD_LANGUAGE")
if langStr != "" {
lang = langCode(langStr)
}
basePath := filepath.Join(localesDir, defaultBaseFile)
b, err := os.ReadFile(basePath)
if err != nil {
return fmt.Errorf("upload: %w", err)
}
var buf bytes.Buffer
buf.Write(b)
uri = translationURL(uploadURI, defaultBaseFile, projectID, lang)
var client http.Client
resp, err := client.Post(uri.String(), "application/json", &buf)
if err != nil {
return fmt.Errorf("upload: client post: %w", err)
}
defer func() {
err = errors.WithDeferred(err, resp.Body.Close())
}()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
}
return nil
}

838
scripts/translations/package-lock.json generated
View File

@ -1,838 +0,0 @@
{
"name": "translations",
"version": "0.2.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"version": "0.2.0",
"dependencies": {
"request": "^2.88.0",
"request-promise": "^4.2.2"
}
},
"node_modules/ajv": {
"version": "6.5.5",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.5.5.tgz",
"integrity": "sha512-7q7gtRQDJSyuEHjuVgHoUa2VuemFiCMrfQc9Tc08XTAc4Zj/5U1buQJ0HU6i7fKjXU09SVgSmxa4sLvuvS8Iyg==",
"dependencies": {
"fast-deep-equal": "^2.0.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
}
},
"node_modules/asn1": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
"integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
"dependencies": {
"safer-buffer": "~2.1.0"
}
},
"node_modules/assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
"engines": {
"node": ">=0.8"
}
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"node_modules/aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=",
"engines": {
"node": "*"
}
},
"node_modules/aws4": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz",
"integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ=="
},
"node_modules/bcrypt-pbkdf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
"integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
"dependencies": {
"tweetnacl": "^0.14.3"
}
},
"node_modules/bluebird": {
"version": "3.5.3",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz",
"integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw=="
},
"node_modules/caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
},
"node_modules/combined-stream": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
"integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==",
"dependencies": {
"delayed-stream": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"node_modules/dashdash": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
"integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
"dependencies": {
"assert-plus": "^1.0.0"
},
"engines": {
"node": ">=0.10"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/ecc-jsbn": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
"integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
"dependencies": {
"jsbn": "~0.1.0",
"safer-buffer": "^2.1.0"
}
},
"node_modules/extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
},
"node_modules/extsprintf": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
"integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=",
"engines": [
"node >=0.6.0"
]
},
"node_modules/fast-deep-equal": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz",
"integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk="
},
"node_modules/fast-json-stable-stringify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
"integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
},
"node_modules/forever-agent": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=",
"engines": {
"node": "*"
}
},
"node_modules/form-data": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.6",
"mime-types": "^2.1.12"
},
"engines": {
"node": ">= 0.12"
}
},
"node_modules/getpass": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
"integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
"dependencies": {
"assert-plus": "^1.0.0"
}
},
"node_modules/har-schema": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
"integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=",
"engines": {
"node": ">=4"
}
},
"node_modules/har-validator": {
"version": "5.1.3",
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz",
"integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==",
"dependencies": {
"ajv": "^6.5.5",
"har-schema": "^2.0.0"
},
"engines": {
"node": ">=6"
}
},
"node_modules/http-signature": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
"integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
"dependencies": {
"assert-plus": "^1.0.0",
"jsprim": "^1.2.2",
"sshpk": "^1.7.0"
},
"engines": {
"node": ">=0.8",
"npm": ">=1.3.7"
}
},
"node_modules/is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
},
"node_modules/isstream": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
},
"node_modules/jsbn": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
},
"node_modules/json-schema": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
"integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
},
"node_modules/json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
},
"node_modules/json-stringify-safe": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
"integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
},
"node_modules/jsprim": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
"integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
"engines": [
"node >=0.6.0"
],
"dependencies": {
"assert-plus": "1.0.0",
"extsprintf": "1.3.0",
"json-schema": "0.2.3",
"verror": "1.10.0"
}
},
"node_modules/lodash": {
"version": "4.17.20",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
},
"node_modules/mime-db": {
"version": "1.37.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz",
"integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime-types": {
"version": "2.1.21",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz",
"integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==",
"dependencies": {
"mime-db": "~1.37.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/oauth-sign": {
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
"engines": {
"node": "*"
}
},
"node_modules/performance-now": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
},
"node_modules/psl": {
"version": "1.1.29",
"resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz",
"integrity": "sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ=="
},
"node_modules/punycode": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
},
"node_modules/qs": {
"version": "6.5.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==",
"engines": {
"node": ">=0.6"
}
},
"node_modules/request": {
"version": "2.88.0",
"resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz",
"integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==",
"dependencies": {
"aws-sign2": "~0.7.0",
"aws4": "^1.8.0",
"caseless": "~0.12.0",
"combined-stream": "~1.0.6",
"extend": "~3.0.2",
"forever-agent": "~0.6.1",
"form-data": "~2.3.2",
"har-validator": "~5.1.0",
"http-signature": "~1.2.0",
"is-typedarray": "~1.0.0",
"isstream": "~0.1.2",
"json-stringify-safe": "~5.0.1",
"mime-types": "~2.1.19",
"oauth-sign": "~0.9.0",
"performance-now": "^2.1.0",
"qs": "~6.5.2",
"safe-buffer": "^5.1.2",
"tough-cookie": "~2.4.3",
"tunnel-agent": "^0.6.0",
"uuid": "^3.3.2"
},
"engines": {
"node": ">= 4"
}
},
"node_modules/request-promise": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.2.tgz",
"integrity": "sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=",
"dependencies": {
"bluebird": "^3.5.0",
"request-promise-core": "1.1.1",
"stealthy-require": "^1.1.0",
"tough-cookie": ">=2.3.3"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/request-promise-core": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz",
"integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=",
"dependencies": {
"lodash": "^4.13.1"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"node_modules/safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"node_modules/sshpk": {
"version": "1.15.2",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.15.2.tgz",
"integrity": "sha512-Ra/OXQtuh0/enyl4ETZAfTaeksa6BXks5ZcjpSUNrjBr0DvrJKX+1fsKDPpT9TBXgHAFsa4510aNVgI8g/+SzA==",
"dependencies": {
"asn1": "~0.2.3",
"assert-plus": "^1.0.0",
"bcrypt-pbkdf": "^1.0.0",
"dashdash": "^1.12.0",
"ecc-jsbn": "~0.1.1",
"getpass": "^0.1.1",
"jsbn": "~0.1.0",
"safer-buffer": "^2.0.2",
"tweetnacl": "~0.14.0"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/stealthy-require": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
"integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/tough-cookie": {
"version": "2.4.3",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz",
"integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==",
"dependencies": {
"psl": "^1.1.24",
"punycode": "^1.4.1"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/tunnel-agent": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
"dependencies": {
"safe-buffer": "^5.0.1"
},
"engines": {
"node": "*"
}
},
"node_modules/tweetnacl": {
"version": "0.14.5",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
},
"node_modules/uri-js": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
"integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
"dependencies": {
"punycode": "^2.1.0"
}
},
"node_modules/uri-js/node_modules/punycode": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
"engines": {
"node": ">=6"
}
},
"node_modules/uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==",
"bin": {
"uuid": "bin/uuid"
}
},
"node_modules/verror": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
"integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
"engines": [
"node >=0.6.0"
],
"dependencies": {
"assert-plus": "^1.0.0",
"core-util-is": "1.0.2",
"extsprintf": "^1.2.0"
}
}
},
"dependencies": {
"ajv": {
"version": "6.5.5",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.5.5.tgz",
"integrity": "sha512-7q7gtRQDJSyuEHjuVgHoUa2VuemFiCMrfQc9Tc08XTAc4Zj/5U1buQJ0HU6i7fKjXU09SVgSmxa4sLvuvS8Iyg==",
"requires": {
"fast-deep-equal": "^2.0.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
}
},
"asn1": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
"integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
"requires": {
"safer-buffer": "~2.1.0"
}
},
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
},
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
},
"aws4": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz",
"integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ=="
},
"bcrypt-pbkdf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
"integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
"requires": {
"tweetnacl": "^0.14.3"
}
},
"bluebird": {
"version": "3.5.3",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz",
"integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw=="
},
"caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
},
"combined-stream": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
"integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==",
"requires": {
"delayed-stream": "~1.0.0"
}
},
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"dashdash": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
"integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
"requires": {
"assert-plus": "^1.0.0"
}
},
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
},
"ecc-jsbn": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
"integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
"requires": {
"jsbn": "~0.1.0",
"safer-buffer": "^2.1.0"
}
},
"extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
},
"extsprintf": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
"integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
},
"fast-deep-equal": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz",
"integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk="
},
"fast-json-stable-stringify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
"integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
},
"forever-agent": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
},
"form-data": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.6",
"mime-types": "^2.1.12"
}
},
"getpass": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
"integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
"requires": {
"assert-plus": "^1.0.0"
}
},
"har-schema": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
"integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
},
"har-validator": {
"version": "5.1.3",
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz",
"integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==",
"requires": {
"ajv": "^6.5.5",
"har-schema": "^2.0.0"
}
},
"http-signature": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
"integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
"requires": {
"assert-plus": "^1.0.0",
"jsprim": "^1.2.2",
"sshpk": "^1.7.0"
}
},
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
},
"isstream": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
},
"jsbn": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
},
"json-schema": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
"integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
},
"json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
},
"json-stringify-safe": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
"integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
},
"jsprim": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
"integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
"requires": {
"assert-plus": "1.0.0",
"extsprintf": "1.3.0",
"json-schema": "0.2.3",
"verror": "1.10.0"
}
},
"lodash": {
"version": "4.17.20",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
},
"mime-db": {
"version": "1.37.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz",
"integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg=="
},
"mime-types": {
"version": "2.1.21",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz",
"integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==",
"requires": {
"mime-db": "~1.37.0"
}
},
"oauth-sign": {
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ=="
},
"performance-now": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
},
"psl": {
"version": "1.1.29",
"resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz",
"integrity": "sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ=="
},
"punycode": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
},
"qs": {
"version": "6.5.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
},
"request": {
"version": "2.88.0",
"resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz",
"integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==",
"requires": {
"aws-sign2": "~0.7.0",
"aws4": "^1.8.0",
"caseless": "~0.12.0",
"combined-stream": "~1.0.6",
"extend": "~3.0.2",
"forever-agent": "~0.6.1",
"form-data": "~2.3.2",
"har-validator": "~5.1.0",
"http-signature": "~1.2.0",
"is-typedarray": "~1.0.0",
"isstream": "~0.1.2",
"json-stringify-safe": "~5.0.1",
"mime-types": "~2.1.19",
"oauth-sign": "~0.9.0",
"performance-now": "^2.1.0",
"qs": "~6.5.2",
"safe-buffer": "^5.1.2",
"tough-cookie": "~2.4.3",
"tunnel-agent": "^0.6.0",
"uuid": "^3.3.2"
}
},
"request-promise": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.2.tgz",
"integrity": "sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=",
"requires": {
"bluebird": "^3.5.0",
"request-promise-core": "1.1.1",
"stealthy-require": "^1.1.0",
"tough-cookie": ">=2.3.3"
}
},
"request-promise-core": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz",
"integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=",
"requires": {
"lodash": "^4.13.1"
}
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"sshpk": {
"version": "1.15.2",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.15.2.tgz",
"integrity": "sha512-Ra/OXQtuh0/enyl4ETZAfTaeksa6BXks5ZcjpSUNrjBr0DvrJKX+1fsKDPpT9TBXgHAFsa4510aNVgI8g/+SzA==",
"requires": {
"asn1": "~0.2.3",
"assert-plus": "^1.0.0",
"bcrypt-pbkdf": "^1.0.0",
"dashdash": "^1.12.0",
"ecc-jsbn": "~0.1.1",
"getpass": "^0.1.1",
"jsbn": "~0.1.0",
"safer-buffer": "^2.0.2",
"tweetnacl": "~0.14.0"
}
},
"stealthy-require": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
"integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
},
"tough-cookie": {
"version": "2.4.3",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz",
"integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==",
"requires": {
"psl": "^1.1.24",
"punycode": "^1.4.1"
}
},
"tunnel-agent": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
"requires": {
"safe-buffer": "^5.0.1"
}
},
"tweetnacl": {
"version": "0.14.5",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
},
"uri-js": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
"integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
"requires": {
"punycode": "^2.1.0"
},
"dependencies": {
"punycode": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
}
}
},
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
},
"verror": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
"integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
"requires": {
"assert-plus": "^1.0.0",
"core-util-is": "1.0.2",
"extsprintf": "^1.2.0"
}
}
}
}

14
scripts/translations/package.json
View File

@ -1,14 +0,0 @@
{
"name": "translations",
"version": "0.2.0",
"scripts": {
"locales:download": "TWOSKY_URI=https://twosky.int.agrd.dev/api/v1 TWOSKY_PROJECT_ID=home node download.js ; node count.js",
"locales:upload": "TWOSKY_URI=https://twosky.int.agrd.dev/api/v1 TWOSKY_PROJECT_ID=home node upload.js",
"locales:summary": "node count.js",
"locales:unused": "node unused.js"
},
"dependencies": {
"request": "^2.88.0",
"request-promise": "^4.2.2"
}
}

63
scripts/translations/unused.js
View File

@ -1,63 +0,0 @@
const fs = require('fs');
const path = require('path');
const SRC_DIR = '../../client/src/'
const LOCALES_DIR = '../../client/src/__locales';
const BASE_FILE = path.join(LOCALES_DIR, 'en.json');
// Strings that may be found by the algorithm,
// but in fact they are used.
const KNOWN_USED_STRINGS = {
'blocking_mode_refused': true,
'blocking_mode_nxdomain': true,
'blocking_mode_custom_ip': true,
}
function traverseDir(dir, callback) {
fs.readdirSync(dir).forEach(file => {
let fullPath = path.join(dir, file);
if (fs.lstatSync(fullPath).isDirectory()) {
traverseDir(fullPath, callback);
} else {
callback(fullPath);
}
});
}
const contains = (key, files) => {
for (let file of files) {
if (file.includes(key)) {
return true;
}
}
return false;
}
const main = () => {
const baseLanguage = require(BASE_FILE);
const files = [];
traverseDir(SRC_DIR, (path) => {
const canContain = (path.endsWith('.js') || path.endsWith('.json')) &&
!path.includes(LOCALES_DIR);
if (canContain) {
files.push(fs.readFileSync(path).toString());
}
});
const unused = [];
for (let key in baseLanguage) {
if (!contains(key, files) && !KNOWN_USED_STRINGS[key]) {
unused.push(key);
}
}
console.log('Unused keys:');
for (let key of unused) {
console.log(key);
}
}
main();

47
scripts/translations/upload.js
View File

@ -1,47 +0,0 @@
const path = require('path');
const fs = require('fs');
const request = require('request-promise');
const twoskyConfig = require('../../.twosky.json')[0];
const { project_id: TWOSKY_PROJECT_ID, base_locale: DEFAULT_LANGUAGE } = twoskyConfig;
const LOCALES_DIR = '../../client/src/__locales';
const BASE_FILE = 'en.json';
const TWOSKY_URI = process.env.TWOSKY_URI;
/**
* Prepare post params
*/
const getRequestData = (url, projectId) => {
const language = process.env.UPLOAD_LANGUAGE || DEFAULT_LANGUAGE;
const formData = {
format: 'json',
language: language,
filename: BASE_FILE,
project: projectId,
file: fs.createReadStream(path.resolve(LOCALES_DIR, `${language}.json`)),
};
console.log(`uploading ${language}`);
return {
url: `${url}/upload`,
formData
};
};
/**
* Make request to twosky to upload new json
*/
const upload = () => {
if (!TWOSKY_URI) {
console.error('No credentials');
return;
}
const { url, formData } = getRequestData(TWOSKY_URI, TWOSKY_PROJECT_ID);
request
.post({ url, formData })
.catch(err => console.log(err));
};
upload();