Pull request: 3020 runtime clients sources control

Merge in DNS/adguard-home from 3020-client-sources to master

Closes #3020.

Squashed commit of the following:

commit f8e6b6d63373f99b52f7b8c32f4242c453daf1a4
Merge: 41fb071d 0a1ff65b
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Apr 25 19:19:15 2022 +0300

    Merge branch 'master' into 3020-client-sources

commit 41fb071deb2a87e0a69d09c8f418a016b4dd7e93
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Apr 25 13:38:28 2022 +0300

    home: fix nil, imp docs

commit aaa7765914a8a4645eba357cd088a9470611ffdc
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Apr 25 12:25:47 2022 +0300

    home: imp code

commit 3f71b999564c604583b46313d29f5b70cf51ee14
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Apr 22 19:12:27 2022 +0300

    home: runtime clients sources control
This commit is contained in:
Eugene Burkov 2022-04-26 13:04:16 +03:00
parent 0a1ff65b4a
commit 235316e050
8 changed files with 283 additions and 65 deletions

View File

@ -23,6 +23,8 @@ and this project adheres to
### Added ### Added
- The ability to control each source of runtime clients separately via
`clients.runtime_sources` configuration object ([#3020]).
- The ability to customize the set of networks that are considered private - The ability to customize the set of networks that are considered private
through the new `dns.private_networks` property in the configuration file through the new `dns.private_networks` property in the configuration file
([#3142]). ([#3142]).
@ -63,8 +65,36 @@ and this project adheres to
#### Configuration Changes #### Configuration Changes
In this release, the schema version has changed from 12 to 13. In this release, the schema version has changed from 12 to 14.
- Object `clients`, which in schema versions 13 and earlier was an array of
actual persistent clients, is now consist of `persistent` and
`runtime_sources` properties:
```yaml
# BEFORE:
'clients':
- name: client-name
# …
# AFTER:
'clients':
'persistent':
- name: client-name
# …
'runtime_sources':
whois: true
arp: true
rdns: true
dhcp: true
hosts: true
```
The value for `clients.runtime_sources.rdns` field is taken from
`dns.resolve_clients` property. To rollback this change, remove the
`runtime_sources` property, move the contents of `persistent` into the
`clients` itself, the value of `clients.runtime_sources.rdns` into the
`dns.resolve_clietns`, and change the `schema_version` back to `13`.
- Property `local_domain_name`, which in schema versions 12 and earlier used to - Property `local_domain_name`, which in schema versions 12 and earlier used to
be a part of the `dns` object, is now a part of the `dhcp` object: be a part of the `dns` object, is now a part of the `dhcp` object:
@ -85,6 +115,9 @@ In this release, the schema version has changed from 12 to 13.
### Deprecated ### Deprecated
- The `--no-etc-hosts` option. Its' functionality is now controlled by
`clients.runtime_sources.hosts` configuration property. v0.109.0 will remove
the flag completely.
- Go 1.17 support. v0.109.0 will require at least Go 1.18 to build. - Go 1.17 support. v0.109.0 will require at least Go 1.18 to build.
### Fixed ### Fixed
@ -94,6 +127,7 @@ In this release, the schema version has changed from 12 to 13.
[#1730]: https://github.com/AdguardTeam/AdGuardHome/issues/1730 [#1730]: https://github.com/AdguardTeam/AdGuardHome/issues/1730
[#2993]: https://github.com/AdguardTeam/AdGuardHome/issues/2993 [#2993]: https://github.com/AdguardTeam/AdGuardHome/issues/2993
[#3020]: https://github.com/AdguardTeam/AdGuardHome/issues/3020
[#3057]: https://github.com/AdguardTeam/AdGuardHome/issues/3057 [#3057]: https://github.com/AdguardTeam/AdGuardHome/issues/3057
[#3142]: https://github.com/AdguardTeam/AdGuardHome/issues/3142 [#3142]: https://github.com/AdguardTeam/AdGuardHome/issues/3142
[#3157]: https://github.com/AdguardTeam/AdGuardHome/issues/3157 [#3157]: https://github.com/AdguardTeam/AdGuardHome/issues/3157

View File

@ -59,6 +59,16 @@ const (
ClientSourceHostsFile ClientSourceHostsFile
) )
// clientSourceConf is used to configure where the runtime clients will be
// obtained from.
type clientSourcesConf struct {
WHOIS bool `yaml:"whois"`
ARP bool `yaml:"arp"`
RDNS bool `yaml:"rdns"`
DHCP bool `yaml:"dhcp"`
HostsFile bool `yaml:"hosts"`
}
// RuntimeClient information // RuntimeClient information
type RuntimeClient struct { type RuntimeClient struct {
WHOISInfo *RuntimeClientWHOISInfo WHOISInfo *RuntimeClientWHOISInfo
@ -134,14 +144,14 @@ func (clients *clientsContainer) Init(
clients.dhcpServer.SetOnLeaseChanged(clients.onDHCPLeaseChanged) clients.dhcpServer.SetOnLeaseChanged(clients.onDHCPLeaseChanged)
} }
go clients.handleHostsUpdates() if clients.etcHosts != nil {
go clients.handleHostsUpdates()
}
} }
func (clients *clientsContainer) handleHostsUpdates() { func (clients *clientsContainer) handleHostsUpdates() {
if clients.etcHosts != nil { for upd := range clients.etcHosts.Upd() {
for upd := range clients.etcHosts.Upd() { clients.addFromHostsFile(upd)
clients.addFromHostsFile(upd)
}
} }
} }
@ -158,7 +168,9 @@ func (clients *clientsContainer) Start() {
// Reload reloads runtime clients. // Reload reloads runtime clients.
func (clients *clientsContainer) Reload() { func (clients *clientsContainer) Reload() {
clients.addFromSystemARP() if clients.arpdb != nil {
clients.addFromSystemARP()
}
} }
type clientObject struct { type clientObject struct {
@ -843,7 +855,7 @@ func (clients *clientsContainer) addFromSystemARP() {
// updateFromDHCP adds the clients that have a non-empty hostname from the DHCP // updateFromDHCP adds the clients that have a non-empty hostname from the DHCP
// server. // server.
func (clients *clientsContainer) updateFromDHCP(add bool) { func (clients *clientsContainer) updateFromDHCP(add bool) {
if clients.dhcpServer == nil { if clients.dhcpServer == nil || !config.Clients.Sources.DHCP {
return return
} }

View File

@ -51,6 +51,13 @@ type osConfig struct {
RlimitNoFile uint64 `yaml:"rlimit_nofile"` RlimitNoFile uint64 `yaml:"rlimit_nofile"`
} }
type clientsConfig struct {
// Sources defines the set of sources to fetch the runtime clients from.
Sources *clientSourcesConf `yaml:"runtime_sources"`
// Persistent are the configured clients.
Persistent []*clientObject `yaml:"persistent"`
}
// configuration is loaded from YAML // configuration is loaded from YAML
// field ordering is important -- yaml fields will mirror ordering from here // field ordering is important -- yaml fields will mirror ordering from here
type configuration struct { type configuration struct {
@ -88,7 +95,7 @@ type configuration struct {
// Clients contains the YAML representations of the persistent clients. // Clients contains the YAML representations of the persistent clients.
// This field is only used for reading and writing persistent client data. // This field is only used for reading and writing persistent client data.
// Keep this field sorted to ensure consistent ordering. // Keep this field sorted to ensure consistent ordering.
Clients []*clientObject `yaml:"clients"` Clients *clientsConfig `yaml:"clients"`
logSettings `yaml:",inline"` logSettings `yaml:",inline"`
@ -123,9 +130,6 @@ type dnsConfig struct {
// UpstreamTimeout is the timeout for querying upstream servers. // UpstreamTimeout is the timeout for querying upstream servers.
UpstreamTimeout timeutil.Duration `yaml:"upstream_timeout"` UpstreamTimeout timeutil.Duration `yaml:"upstream_timeout"`
// ResolveClients enables and disables resolving clients with RDNS.
ResolveClients bool `yaml:"resolve_clients"`
// PrivateNets is the set of IP networks for which the private reverse DNS // PrivateNets is the set of IP networks for which the private reverse DNS
// resolver should be used. // resolver should be used.
PrivateNets []string `yaml:"private_networks"` PrivateNets []string `yaml:"private_networks"`
@ -198,7 +202,6 @@ var config = &configuration{
FilteringEnabled: true, // whether or not use filter lists FilteringEnabled: true, // whether or not use filter lists
FiltersUpdateIntervalHours: 24, FiltersUpdateIntervalHours: 24,
UpstreamTimeout: timeutil.Duration{Duration: dnsforward.DefaultTimeout}, UpstreamTimeout: timeutil.Duration{Duration: dnsforward.DefaultTimeout},
ResolveClients: true,
UsePrivateRDNS: true, UsePrivateRDNS: true,
}, },
TLS: tlsConfigSettings{ TLS: tlsConfigSettings{
@ -209,6 +212,15 @@ var config = &configuration{
DHCP: &dhcpd.ServerConfig{ DHCP: &dhcpd.ServerConfig{
LocalDomainName: "lan", LocalDomainName: "lan",
}, },
Clients: &clientsConfig{
Sources: &clientSourcesConf{
WHOIS: true,
ARP: true,
RDNS: true,
DHCP: true,
HostsFile: true,
},
},
logSettings: logSettings{ logSettings: logSettings{
LogCompress: false, LogCompress: false,
LogLocalTime: false, LogLocalTime: false,
@ -404,9 +416,7 @@ func (c *configuration) write() error {
s.WriteDiskConfig(&c) s.WriteDiskConfig(&c)
dns := &config.DNS dns := &config.DNS
dns.FilteringConfig = c dns.FilteringConfig = c
dns.LocalPTRResolvers, dns.LocalPTRResolvers, config.Clients.Sources.RDNS, dns.UsePrivateRDNS = s.RDNSSettings()
dns.ResolveClients,
dns.UsePrivateRDNS = s.RDNSSettings()
} }
if Context.dhcpServer != nil { if Context.dhcpServer != nil {
@ -415,7 +425,7 @@ func (c *configuration) write() error {
config.DHCP = c config.DHCP = c
} }
config.Clients = Context.clients.forConfig() config.Clients.Persistent = Context.clients.forConfig()
configFile := config.getConfigFilename() configFile := config.getConfigFilename()
log.Debug("Writing YAML file: %s", configFile) log.Debug("Writing YAML file: %s", configFile)

View File

@ -136,7 +136,10 @@ func initDNSServer() (err error) {
} }
Context.rdns = NewRDNS(Context.dnsServer, &Context.clients, config.DNS.UsePrivateRDNS) Context.rdns = NewRDNS(Context.dnsServer, &Context.clients, config.DNS.UsePrivateRDNS)
Context.whois = initWHOIS(&Context.clients)
if !config.Clients.Sources.WHOIS {
Context.whois = initWHOIS(&Context.clients)
}
Context.filters.Init() Context.filters.Init()
return nil return nil
@ -153,10 +156,11 @@ func onDNSRequest(pctx *proxy.DNSContext) {
return return
} }
if config.DNS.ResolveClients && !ip.IsLoopback() { srcs := config.Clients.Sources
if srcs.RDNS && !ip.IsLoopback() {
Context.rdns.Begin(ip) Context.rdns.Begin(ip)
} }
if !netutil.IsSpecialPurpose(ip) { if srcs.WHOIS && !netutil.IsSpecialPurpose(ip) {
Context.whois.Begin(ip) Context.whois.Begin(ip)
} }
} }
@ -239,7 +243,7 @@ func generateServerConfig() (newConf dnsforward.ServerConfig, err error) {
newConf.FilterHandler = applyAdditionalFiltering newConf.FilterHandler = applyAdditionalFiltering
newConf.GetCustomUpstreamByClient = Context.clients.findUpstreams newConf.GetCustomUpstreamByClient = Context.clients.findUpstreams
newConf.ResolveClients = dnsConf.ResolveClients newConf.ResolveClients = config.Clients.Sources.RDNS
newConf.UsePrivateRDNS = dnsConf.UsePrivateRDNS newConf.UsePrivateRDNS = dnsConf.UsePrivateRDNS
newConf.LocalPTRResolvers = dnsConf.LocalPTRResolvers newConf.LocalPTRResolvers = dnsConf.LocalPTRResolvers
newConf.UpstreamTimeout = dnsConf.UpstreamTimeout.Duration newConf.UpstreamTimeout = dnsConf.UpstreamTimeout.Duration
@ -387,10 +391,11 @@ func startDNSServer() error {
continue continue
} }
if config.DNS.ResolveClients && !ip.IsLoopback() { srcs := config.Clients.Sources
if srcs.RDNS && !ip.IsLoopback() {
Context.rdns.Begin(ip) Context.rdns.Begin(ip)
} }
if !netutil.IsSpecialPurpose(ip) { if srcs.WHOIS && !netutil.IsSpecialPurpose(ip) {
Context.whois.Begin(ip) Context.whois.Begin(ip)
} }
} }

View File

@ -173,6 +173,11 @@ func setupContext(args options) {
os.Exit(0) os.Exit(0)
} }
if !args.noEtcHosts && config.Clients.Sources.HostsFile {
err = setupHostsContainer()
fatalOnError(err)
}
} }
Context.mux = http.NewServeMux() Context.mux = http.NewServeMux()
@ -285,14 +290,12 @@ func setupConfig(args options) (err error) {
ConfName: config.getConfigFilename(), ConfName: config.getConfigFilename(),
}) })
if !args.noEtcHosts { var arpdb aghnet.ARPDB
if err = setupHostsContainer(); err != nil { if config.Clients.Sources.ARP {
return err arpdb = aghnet.NewARPDB()
}
} }
arpdb := aghnet.NewARPDB() Context.clients.Init(config.Clients.Persistent, Context.dhcpServer, Context.etcHosts, arpdb)
Context.clients.Init(config.Clients, Context.dhcpServer, Context.etcHosts, arpdb)
if args.bindPort != 0 { if args.bindPort != 0 {
uc := aghalg.UniqChecker{} uc := aghalg.UniqChecker{}

View File

@ -230,13 +230,19 @@ var helpArg = arg{
} }
var noEtcHostsArg = arg{ var noEtcHostsArg = arg{
description: "Do not use the OS-provided hosts.", description: "Deprecated. Do not use the OS-provided hosts.",
longName: "no-etc-hosts", longName: "no-etc-hosts",
shortName: "", shortName: "",
updateWithValue: nil, updateWithValue: nil,
updateNoValue: func(o options) (options, error) { o.noEtcHosts = true; return o, nil }, updateNoValue: func(o options) (options, error) { o.noEtcHosts = true; return o, nil },
effect: nil, effect: func(_ options, _ string) (f effect, err error) {
serialize: func(o options) []string { return boolSliceOrNil(o.noEtcHosts) }, log.Info(
"warning: --no-etc-hosts flag is deprecated and will be removed in the future versions",
)
return nil, nil
},
serialize: func(o options) []string { return boolSliceOrNil(o.noEtcHosts) },
} }
var localFrontendArg = arg{ var localFrontendArg = arg{

View File

@ -21,9 +21,11 @@ import (
) )
// currentSchemaVersion is the current schema version. // currentSchemaVersion is the current schema version.
const currentSchemaVersion = 13 const currentSchemaVersion = 14
// These aliases are provided for convenience. // These aliases are provided for convenience.
//
// TODO(e.burkov): Remove any after updating to Go 1.18.
type ( type (
any = interface{} any = interface{}
yarr = []any yarr = []any
@ -86,6 +88,7 @@ func upgradeConfigSchema(oldVersion int, diskConf yobj) (err error) {
upgradeSchema10to11, upgradeSchema10to11,
upgradeSchema11to12, upgradeSchema11to12,
upgradeSchema12to13, upgradeSchema12to13,
upgradeSchema13to14,
} }
n := 0 n := 0
@ -726,7 +729,7 @@ func upgradeSchema12to13(diskConf yobj) (err error) {
var dhcp yobj var dhcp yobj
dhcp, ok = dhcpVal.(yobj) dhcp, ok = dhcpVal.(yobj)
if !ok { if !ok {
return fmt.Errorf("unexpected type of dhcp: %T", dnsVal) return fmt.Errorf("unexpected type of dhcp: %T", dhcpVal)
} }
const field = "local_domain_name" const field = "local_domain_name"
@ -737,6 +740,68 @@ func upgradeSchema12to13(diskConf yobj) (err error) {
return nil return nil
} }
// upgradeSchema13to14 performs the following changes:
//
// # BEFORE:
// 'clients':
// - 'name': 'client-name'
// # …
//
// # AFTER:
// 'clients':
// 'persistent':
// - 'name': 'client-name'
// # …
// 'runtime_sources':
// 'whois': true
// 'arp': true
// 'rdns': true
// 'dhcp': true
// 'hosts': true
//
func upgradeSchema13to14(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 13 to 14")
diskConf["schema_version"] = 14
clientsVal, ok := diskConf["clients"]
if !ok {
clientsVal = yarr{}
}
var rdnsSrc bool
if dnsVal, dok := diskConf["dns"]; dok {
var dnsSettings yobj
dnsSettings, ok = dnsVal.(yobj)
if !ok {
return fmt.Errorf("unexpected type of dns: %T", dnsVal)
}
var rdnsSrcVal any
rdnsSrcVal, ok = dnsSettings["resolve_clients"]
if ok {
rdnsSrc, ok = rdnsSrcVal.(bool)
if !ok {
return fmt.Errorf("unexpected type of resolve_clients: %T", rdnsSrcVal)
}
delete(dnsSettings, "resolve_clients")
}
}
diskConf["clients"] = yobj{
"persistent": clientsVal,
"runtime_sources": &clientSourcesConf{
WHOIS: true,
ARP: true,
RDNS: rdnsSrc,
DHCP: true,
HostsFile: true,
},
}
return nil
}
// TODO(a.garipov): Replace with log.Output when we port it to our logging // TODO(a.garipov): Replace with log.Output when we port it to our logging
// package. // package.
func funcName() string { func funcName() string {

View File

@ -513,46 +513,129 @@ func TestUpgradeSchema11to12(t *testing.T) {
} }
func TestUpgradeSchema12to13(t *testing.T) { func TestUpgradeSchema12to13(t *testing.T) {
t.Run("no_dns", func(t *testing.T) { const newSchemaVer = 13
conf := yobj{}
err := upgradeSchema12to13(conf) testCases := []struct {
require.NoError(t, err) in yobj
want yobj
assert.Equal(t, conf["schema_version"], 13) name string
}) }{{
in: yobj{},
t.Run("no_dhcp", func(t *testing.T) { want: yobj{"schema_version": newSchemaVer},
conf := yobj{ name: "no_dns",
"dns": yobj{}, }, {
} in: yobj{"dns": yobj{}},
want: yobj{
err := upgradeSchema12to13(conf) "dns": yobj{},
require.NoError(t, err) "schema_version": newSchemaVer,
},
assert.Equal(t, conf["schema_version"], 13) name: "no_dhcp",
}) }, {
in: yobj{
t.Run("good", func(t *testing.T) {
conf := yobj{
"dns": yobj{ "dns": yobj{
"local_domain_name": "lan", "local_domain_name": "lan",
}, },
"dhcp": yobj{}, "dhcp": yobj{},
"schema_version": 12, "schema_version": newSchemaVer - 1,
} },
want: yobj{
wantConf := yobj{
"dns": yobj{}, "dns": yobj{},
"dhcp": yobj{ "dhcp": yobj{
"local_domain_name": "lan", "local_domain_name": "lan",
}, },
"schema_version": 13, "schema_version": newSchemaVer,
} },
name: "good",
}}
err := upgradeSchema12to13(conf) for _, tc := range testCases {
require.NoError(t, err) t.Run(tc.name, func(t *testing.T) {
err := upgradeSchema12to13(tc.in)
require.NoError(t, err)
assert.Equal(t, wantConf, conf) assert.Equal(t, tc.want, tc.in)
}) })
}
}
func TestUpgradeSchema13to14(t *testing.T) {
const newSchemaVer = 14
testClient := &clientObject{
Name: "agh-client",
IDs: []string{"id1"},
UseGlobalSettings: true,
}
testCases := []struct {
in yobj
want yobj
name string
}{{
in: yobj{},
want: yobj{
"schema_version": newSchemaVer,
// The clients field will be added anyway.
"clients": yobj{
"persistent": yarr{},
"runtime_sources": &clientSourcesConf{
WHOIS: true,
ARP: true,
RDNS: false,
DHCP: true,
HostsFile: true,
},
},
},
name: "no_clients",
}, {
in: yobj{
"clients": []*clientObject{testClient},
},
want: yobj{
"schema_version": newSchemaVer,
"clients": yobj{
"persistent": []*clientObject{testClient},
"runtime_sources": &clientSourcesConf{
WHOIS: true,
ARP: true,
RDNS: false,
DHCP: true,
HostsFile: true,
},
},
},
name: "no_dns",
}, {
in: yobj{
"clients": []*clientObject{testClient},
"dns": yobj{
"resolve_clients": true,
},
},
want: yobj{
"schema_version": newSchemaVer,
"clients": yobj{
"persistent": []*clientObject{testClient},
"runtime_sources": &clientSourcesConf{
WHOIS: true,
ARP: true,
RDNS: true,
DHCP: true,
HostsFile: true,
},
},
"dns": yobj{},
},
name: "good",
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
err := upgradeSchema13to14(tc.in)
require.NoError(t, err)
assert.Equal(t, tc.want, tc.in)
})
}
} }