From 7931e50673ce0251fb632cd62fefdfcadf002988 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Wed, 2 Sep 2020 14:13:45 +0300 Subject: [PATCH] + DNS: add "ipset" configuration setting Close #1191 Squashed commit of the following: commit ba14b53f9e3d98ad8127aa3af1def0da4269e8c4 Merge: 362f4c44 6b614295 Author: Simon Zolin Date: Wed Sep 2 14:03:19 2020 +0300 Merge remote-tracking branch 'origin/master' into 1191-ipset commit 362f4c44915cb8946db2e80f9a3f5afd74fe5de1 Author: Simon Zolin Date: Wed Sep 2 12:50:56 2020 +0300 minor commit 28e12459166fe3d13fb0dbe59ac11b7d86adb9b4 Author: Simon Zolin Date: Wed Sep 2 12:43:25 2020 +0300 minor commit bdbd7324501f6111bea1e91eda7d730c7ea57b11 Author: Simon Zolin Date: Tue Sep 1 18:40:04 2020 +0300 move code, ipset-v6 commit 77f4d943e74b70b5bc5aea279875ab1e2fab2192 Author: Simon Zolin Date: Tue Sep 1 15:53:27 2020 +0300 comment commit 16401325bbefeba08e447257b12a8424b78c9475 Author: Simon Zolin Date: Mon Aug 31 17:43:23 2020 +0300 minor commit c8410e9a519b87911bc50f504e8b4aaf8dce6e02 Author: Simon Zolin Date: Mon Aug 31 15:30:52 2020 +0300 + DNS: add "ipset" configuration setting --- AGHTechDoc.md | 23 +++++++ dnsforward/config.go | 5 ++ dnsforward/dnsforward.go | 22 ++++--- dnsforward/handle_dns.go | 1 + dnsforward/ipset.go | 133 +++++++++++++++++++++++++++++++++++++++ dnsforward/ipset_test.go | 41 ++++++++++++ 6 files changed, 217 insertions(+), 8 deletions(-) create mode 100644 dnsforward/ipset.go create mode 100644 dnsforward/ipset_test.go diff --git a/AGHTechDoc.md b/AGHTechDoc.md index b0b6ca80..91d17472 100644 --- a/AGHTechDoc.md +++ b/AGHTechDoc.md @@ -69,6 +69,7 @@ Contents: * API: Log out * API: Get current user info * Safe services +* ipset ## Relations between subsystems @@ -1882,3 +1883,25 @@ Check if host name is blocked by SB/PC service: sha256(host.com)[0..1] -> hashes[0],hashes[1],... sha256(sub.host.com)[0..1] -> hashes[2],... ... + + +## ipset + +AGH can add IP addresses of the specified in configuration domain names to an ipset list. + +Prepare: user creates an ipset list and configures AGH for using it. + + 1. User --( ipset create my_ipset hash:ip ) -> OS + 2. User --( ipset: host.com,host2.com/my_ipset )-> AGH + + Syntax: + + ipset: "DOMAIN[,DOMAIN].../IPSET1_NAME[,IPSET2_NAME]..." + + IPv4 addresses are added to an ipset list with `ipv4` family, IPv6 addresses - to `ipv6` ipset list. + +Run-time: AGH adds IP addresses of a domain name to a corresponding ipset list. + + 1. AGH --( resolve host.com )-> upstream + 2. AGH <-( host.com:[1.1.1.1,2.2.2.2] )-- upstream + 3. AGH --( ipset.add(my_ipset, [1.1.1.1,2.2.2.2] ))-> OS diff --git a/dnsforward/config.go b/dnsforward/config.go index db0033c1..69af11eb 100644 --- a/dnsforward/config.go +++ b/dnsforward/config.go @@ -82,6 +82,11 @@ type FilteringConfig struct { EnableDNSSEC bool `yaml:"enable_dnssec"` // Set DNSSEC flag in outcoming DNS request EnableEDNSClientSubnet bool `yaml:"edns_client_subnet"` // Enable EDNS Client Subnet option MaxGoroutines uint32 `yaml:"max_goroutines"` // Max. number of parallel goroutines for processing incoming requests + + // IPSET configuration - add IP addresses of the specified domain names to an ipset list + // Syntax: + // "DOMAIN[,DOMAIN].../IPSET_NAME" + IPSETList []string `yaml:"ipset"` } // TLSConfig is the TLS configuration for HTTPS, DNS-over-HTTPS, and DNS-over-TLS diff --git a/dnsforward/dnsforward.go b/dnsforward/dnsforward.go index 87093879..39d09a39 100644 --- a/dnsforward/dnsforward.go +++ b/dnsforward/dnsforward.go @@ -51,6 +51,8 @@ type Server struct { stats stats.Stats access *accessCtx + ipset ipsetCtx + tableHostToIP map[string]net.IP // "hostname -> IP" table for internal addresses (DHCP) tableHostToIPLock sync.Mutex @@ -168,7 +170,7 @@ func (s *Server) startInternal() error { // Prepare the object func (s *Server) Prepare(config *ServerConfig) error { - // 1. Initialize the server configuration + // Initialize the server configuration // -- if config != nil { s.conf = *config @@ -184,18 +186,22 @@ func (s *Server) Prepare(config *ServerConfig) error { } } - // 2. Set default values in the case if nothing is configured + // Set default values in the case if nothing is configured // -- s.initDefaultSettings() - // 3. Prepare DNS servers settings + // Initialize IPSET configuration + // -- + s.ipset.init(s.conf.IPSETList) + + // Prepare DNS servers settings // -- err := s.prepareUpstreamSettings() if err != nil { return err } - // 3. Create DNS proxy configuration + // Create DNS proxy configuration // -- var proxyConfig proxy.Config proxyConfig, err = s.createProxyConfig() @@ -203,11 +209,11 @@ func (s *Server) Prepare(config *ServerConfig) error { return err } - // 4. Prepare a DNS proxy instance that we use for internal DNS queries + // Prepare a DNS proxy instance that we use for internal DNS queries // -- s.prepareIntlProxy() - // 5. Initialize DNS access module + // Initialize DNS access module // -- s.access = &accessCtx{} err = s.access.Init(s.conf.AllowedClients, s.conf.DisallowedClients, s.conf.BlockedHosts) @@ -215,14 +221,14 @@ func (s *Server) Prepare(config *ServerConfig) error { return err } - // 6. Register web handlers if necessary + // Register web handlers if necessary // -- if !webRegistered && s.conf.HTTPRegister != nil { webRegistered = true s.registerHandlers() } - // 7. Create the main DNS proxy instance + // Create the main DNS proxy instance // -- s.dnsProxy = &proxy.Proxy{Config: proxyConfig} return nil diff --git a/dnsforward/handle_dns.go b/dnsforward/handle_dns.go index 1495e736..3f0f7911 100644 --- a/dnsforward/handle_dns.go +++ b/dnsforward/handle_dns.go @@ -49,6 +49,7 @@ func (s *Server) handleDNSRequest(_ *proxy.Proxy, d *proxy.DNSContext) error { processUpstream, processDNSSECAfterResponse, processFilteringAfterResponse, + s.ipset.process, processQueryLogsAndStats, } for _, process := range mods { diff --git a/dnsforward/ipset.go b/dnsforward/ipset.go new file mode 100644 index 00000000..b07556d5 --- /dev/null +++ b/dnsforward/ipset.go @@ -0,0 +1,133 @@ +package dnsforward + +import ( + "net" + "strings" + + "github.com/AdguardTeam/AdGuardHome/util" + "github.com/AdguardTeam/golibs/log" + "github.com/miekg/dns" +) + +type ipsetCtx struct { + ipsetList map[string][]string // domain -> []ipset_name + ipsetCache map[[4]byte]bool // cache for IP[] to prevent duplicate calls to ipset program + ipset6Cache map[[16]byte]bool // cache for IP[] to prevent duplicate calls to ipset program +} + +// Convert configuration settings to an internal map +// DOMAIN[,DOMAIN].../IPSET1_NAME[,IPSET2_NAME]... +func (c *ipsetCtx) init(ipsetConfig []string) { + c.ipsetList = make(map[string][]string) + c.ipsetCache = make(map[[4]byte]bool) + c.ipset6Cache = make(map[[16]byte]bool) + + for _, it := range ipsetConfig { + it = strings.TrimSpace(it) + hostsAndNames := strings.Split(it, "/") + if len(hostsAndNames) != 2 { + log.Debug("IPSET: invalid value '%s'", it) + continue + } + + ipsetNames := strings.Split(hostsAndNames[1], ",") + if len(ipsetNames) == 0 { + log.Debug("IPSET: invalid value '%s'", it) + continue + } + bad := false + for i := range ipsetNames { + ipsetNames[i] = strings.TrimSpace(ipsetNames[i]) + if len(ipsetNames[i]) == 0 { + bad = true + break + } + } + if bad { + log.Debug("IPSET: invalid value '%s'", it) + continue + } + + hosts := strings.Split(hostsAndNames[0], ",") + for _, host := range hosts { + host = strings.TrimSpace(host) + host = strings.ToLower(host) + if len(host) == 0 { + log.Debug("IPSET: invalid value '%s'", it) + continue + } + c.ipsetList[host] = ipsetNames + } + } + log.Debug("IPSET: added %d hosts", len(c.ipsetList)) +} + +func (c *ipsetCtx) getIP(rr dns.RR) net.IP { + switch a := rr.(type) { + case *dns.A: + var ip4 [4]byte + copy(ip4[:], a.A.To4()) + _, found := c.ipsetCache[ip4] + if found { + return nil // this IP was added before + } + c.ipsetCache[ip4] = false + return a.A + + case *dns.AAAA: + var ip6 [16]byte + copy(ip6[:], a.AAAA) + _, found := c.ipset6Cache[ip6] + if found { + return nil // this IP was added before + } + c.ipset6Cache[ip6] = false + return a.AAAA + + default: + return nil + } +} + +// Add IP addresses of the specified in configuration domain names to an ipset list +func (c *ipsetCtx) process(ctx *dnsContext) int { + req := ctx.proxyCtx.Req + if !(req.Question[0].Qtype == dns.TypeA || + req.Question[0].Qtype == dns.TypeAAAA) || + !ctx.responseFromUpstream { + return resultDone + } + + host := req.Question[0].Name + host = strings.TrimSuffix(host, ".") + host = strings.ToLower(host) + ipsetNames, found := c.ipsetList[host] + if !found { + return resultDone + } + + log.Debug("IPSET: found ipsets %v for host %s", ipsetNames, host) + + for _, it := range ctx.proxyCtx.Res.Answer { + ip := c.getIP(it) + if ip == nil { + continue + } + + ipStr := ip.String() + for _, name := range ipsetNames { + code, out, err := util.RunCommand("ipset", "add", name, ipStr) + if err != nil { + log.Info("IPSET: %s(%s) -> %s: %s", host, ipStr, name, err) + continue + } + if code != 0 { + log.Info("IPSET: ipset add: code:%d output:'%s'", code, out) + continue + } + log.Debug("IPSET: added %s(%s) -> %s", host, ipStr, name) + } + } + + return resultDone +} diff --git a/dnsforward/ipset_test.go b/dnsforward/ipset_test.go new file mode 100644 index 00000000..41be83d2 --- /dev/null +++ b/dnsforward/ipset_test.go @@ -0,0 +1,41 @@ +package dnsforward + +import ( + "testing" + + "github.com/AdguardTeam/dnsproxy/proxy" + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" +) + +func TestIPSET(t *testing.T) { + s := Server{} + s.conf.IPSETList = append(s.conf.IPSETList, "HOST.com/name") + s.conf.IPSETList = append(s.conf.IPSETList, "host2.com,host3.com/name23") + s.conf.IPSETList = append(s.conf.IPSETList, "host4.com/name4,name41") + c := ipsetCtx{} + c.init(s.conf.IPSETList) + + assert.Equal(t, "name", c.ipsetList["host.com"][0]) + assert.Equal(t, "name23", c.ipsetList["host2.com"][0]) + assert.Equal(t, "name23", c.ipsetList["host3.com"][0]) + assert.Equal(t, "name4", c.ipsetList["host4.com"][0]) + assert.Equal(t, "name41", c.ipsetList["host4.com"][1]) + + _, ok := c.ipsetList["host0.com"] + assert.False(t, ok) + + ctx := &dnsContext{ + srv: &s, + } + ctx.proxyCtx = &proxy.DNSContext{} + ctx.proxyCtx.Req = &dns.Msg{ + Question: []dns.Question{ + { + Name: "host.com.", + Qtype: dns.TypeA, + }, + }, + } + assert.Equal(t, resultDone, c.process(ctx)) +}