net/dns/{publicdns,resolver}: add NextDNS DoH support

NextDNS is unique in that users create accounts and then get
user-specific DNS IPs & DoH URLs.

For DoH, the customer ID is in the URL path.

For IPv6, the IP address includes the customer ID in the lower bits.

For IPv4, there's a fragile "IP linking" mechanism to associate your
public IPv4 with an assigned NextDNS IPv4 and that tuple maps to your
customer ID.

We don't use the IP linking mechanism.

Instead, NextDNS is DoH-only. Which means using NextDNS necessarily
shunts all DNS traffic through 100.100.100.100 (programming the OS to
use 100.100.100.100 as the global resolver) because operating systems
can't usually do DoH themselves.

Once it's in Tailscale's DoH client, we then connect out to the known
NextDNS IPv4/IPv6 anycast addresses.

If the control plane sends the client a NextDNS IPv6 address, we then
map it to the corresponding NextDNS DoH with the same client ID, and
we dial that DoH server using the combination of v4/v6 anycast IPs.

Updates #2452

Change-Id: I3439d798d21d5fc9df5a2701839910f5bef85463
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

cmd/tailscaled/depaware.txt

@@ -212,7 +212,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/logtail/filch                                  from tailscale.com/logpolicy
      💣 tailscale.com/metrics                                        from tailscale.com/derp+
         tailscale.com/net/dns                                        from tailscale.com/ipn/ipnlocal+
-        tailscale.com/net/dns/publicdns                              from tailscale.com/net/dns/resolver
+        tailscale.com/net/dns/publicdns                              from tailscale.com/net/dns/resolver+
         tailscale.com/net/dns/resolvconffile                         from tailscale.com/net/dns+
         tailscale.com/net/dns/resolver                               from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
@@ -281,7 +281,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnserver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
-   L    tailscale.com/util/strs                                      from tailscale.com/hostinfo
+        tailscale.com/util/strs                                      from tailscale.com/hostinfo+
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/uniq                                      from tailscale.com/wgengine/magicsock
      💣 tailscale.com/util/winutil                                   from tailscale.com/cmd/tailscaled+

net/dns/config.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 	"sort"
 
+	"tailscale.com/net/dns/publicdns"
 	"tailscale.com/net/dns/resolver"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/dnstype"
@@ -78,13 +79,14 @@ func (c Config) hasRoutes() bool {
 }
 
 // hasDefaultIPResolversOnly reports whether the only resolvers in c are
-// DefaultResolvers, and that those resolvers are simple IP addresses.
+// DefaultResolvers, and that those resolvers are simple IP addresses
+// that speak regular port 53 DNS.
 func (c Config) hasDefaultIPResolversOnly() bool {
 	if !c.hasDefaultResolvers() || c.hasRoutes() {
 		return false
 	}
 	for _, r := range c.DefaultResolvers {
-		if ipp, ok := r.IPPort(); !ok || ipp.Port() != 53 {
+		if ipp, ok := r.IPPort(); !ok || ipp.Port() != 53 || publicdns.IPIsDoHOnlyServer(ipp.Addr()) {
 			return false
 		}
 	}

net/dns/manager.go

@@ -194,6 +194,7 @@ func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig
 			routes[suffix] = resolvers
 		}
 	}
+
 	// Similarly, the OS always gets search paths.
 	ocfg.SearchDomains = cfg.SearchDomains
 	if runtime.GOOS == "windows" {

net/dns/manager_test.go

@@ -562,6 +562,30 @@ func TestManager(t *testing.T) {
 					"bradfitz.ts.com.", "2.3.4.5"),
 			},
 		},
+		{
+			name: "corp-v6",
+			in: Config{
+				DefaultResolvers: mustRes("1::1"),
+			},
+			os: OSConfig{
+				Nameservers: mustIPs("1::1"),
+			},
+		},
+		{
+			// This one's structurally the same as the previous one (corp-v6), but
+			// instead of 1::1 as the IPv6 address, it uses a NextDNS IPv6 address which
+			// is specially recognized.
+			name: "corp-v6-nextdns",
+			in: Config{
+				DefaultResolvers: mustRes("2a07:a8c0::c3:a884"),
+			},
+			os: OSConfig{
+				Nameservers: mustIPs("100.100.100.100"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(".", "2a07:a8c0::c3:a884"),
+			},
+		},
 	}
 
 	trIP := cmp.Transformer("ipStr", func(ip netip.Addr) string { return ip.String() })

net/dns/publicdns/publicdns.go

@@ -7,26 +7,98 @@
 package publicdns
 
 import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
 	"net/netip"
+	"sort"
+	"strings"
 	"sync"
+
+	"tailscale.com/util/strs"
 )
 
-var knownDoH = map[netip.Addr]string{} // 8.8.8.8 => "https://..."
+// dohOfIP maps from public DNS IPs to their DoH base URL.
+//
+// This does not include NextDNS which is handled specially.
+var dohOfIP = map[netip.Addr]string{} // 8.8.8.8 => "https://..."
+
 var dohIPsOfBase = map[string][]netip.Addr{}
 var populateOnce sync.Once
 
-// KnownDoH returns a map of well-known public DNS IPs to their DoH URL.
+// DoHEndpointFromIP returns the DNS-over-HTTPS base URL for a given IP
-// The returned map should not be modified.
+// and whether it's DoH-only (not speaking DNS on port 53).
-func KnownDoH() map[netip.Addr]string {
+//
+// The ok result is whether the IP is a known DNS server.
+func DoHEndpointFromIP(ip netip.Addr) (dohBase string, dohOnly bool, ok bool) {
 	populateOnce.Do(populate)
-	return knownDoH
+	if b, ok := dohOfIP[ip]; ok {
+		return b, false, true
+	}
+
+	// NextDNS DoH URLs are of the form "https://dns.nextdns.io/c3a884"
+	// where the path component is the lower 8 bytes of the IPv6 address
+	// in lowercase hex without any zero padding.
+	if nextDNSv6RangeA.Contains(ip) || nextDNSv6RangeB.Contains(ip) {
+		a := ip.As16()
+		var sb strings.Builder
+		const base = "https://dns.nextdns.io/"
+		sb.Grow(len(base) + 8)
+		sb.WriteString(base)
+		for _, b := range bytes.TrimLeft(a[8:], "\x00") {
+			fmt.Fprintf(&sb, "%02x", b)
+		}
+		return sb.String(), true, true
+	}
+	return "", false, false
 }
 
-// DoHIPsOfBase returns a map of DNS server IP addresses keyed
+// KnownDoHPrefixes returns the list of DoH base URLs.
-// by their DoH URL. It is the inverse of KnownDoH.
+//
-func DoHIPsOfBase() map[string][]netip.Addr {
+// It returns a new copy each time, sorted. It's meant for tests.
+//
+// It does not include providers that have customer-specific DoH URLs like
+// NextDNS.
+func KnownDoHPrefixes() []string {
 	populateOnce.Do(populate)
-	return dohIPsOfBase
+	ret := make([]string, 0, len(dohIPsOfBase))
+	for b := range dohIPsOfBase {
+		ret = append(ret, b)
+	}
+	sort.Strings(ret)
+	return ret
+}
+
+// DoHIPsOfBase returns the IP addresses to use to dial the provided DoH base
+// URL.
+//
+// It is basically the inverse of DoHEndpointFromIP with the exception that for
+// NextDNS it returns IPv4 addresses that DoHEndpointFromIP doesn't map back.
+func DoHIPsOfBase(dohBase string) []netip.Addr {
+	populateOnce.Do(populate)
+	if s := dohIPsOfBase[dohBase]; len(s) > 0 {
+		return s
+	}
+	if hexStr, ok := strs.CutPrefix(dohBase, "https://dns.nextdns.io/"); ok {
+		// TODO(bradfitz): using the NextDNS anycast addresses works but is not
+		// ideal. Some of their regions have better latency via a non-anycast IP
+		// which we could get by first resolving A/AAAA "dns.nextdns.io" over
+		// DoH using their anycast address. For now we only use the anycast
+		// addresses. The IPv4 IPs we use are just the first one in their ranges.
+		// For IPv6 we put the profile ID in the lower bytes, but that seems just
+		// conventional for them and not required (it'll already be in the DoH path).
+		// (Really we shouldn't use either IPv4 or IPv6 anycast for DoH once we
+		// resolve "dns.nextdns.io".)
+		if b, err := hex.DecodeString(hexStr); err == nil && len(b) <= 8 && len(b) > 0 {
+			return []netip.Addr{
+				nextDNSv4One,
+				nextDNSv4Two,
+				nextDNSv6Gen(nextDNSv6RangeA.Addr(), b),
+				nextDNSv6Gen(nextDNSv6RangeB.Addr(), b),
+			}
+		}
+	}
+	return nil
 }
 
 // DoHV6 returns the first IPv6 DNS address from a given public DNS provider
@@ -45,7 +117,7 @@ func DoHV6(base string) (ip netip.Addr, ok bool) {
 // adds it to both knownDoH and dohIPsOFBase maps.
 func addDoH(ipStr, base string) {
 	ip := netip.MustParseAddr(ipStr)
-	knownDoH[ip] = base
+	dohOfIP[ip] = base
 	dohIPsOfBase[base] = append(dohIPsOfBase[base], ip)
 }
 
@@ -106,3 +178,43 @@ func populate() {
 	addDoH("193.19.108.3", "https://adblock.doh.mullvad.net/dns-query")
 	addDoH("2a07:e340::3", "https://adblock.doh.mullvad.net/dns-query")
 }
+
+var (
+	// The NextDNS IPv6 ranges (primary and secondary). The customer ID is
+	// encoded in the lower bytes and is used (in hex form) as the DoH query
+	// path.
+	nextDNSv6RangeA = netip.MustParsePrefix("2a07:a8c0::/33")
+	nextDNSv6RangeB = netip.MustParsePrefix("2a07:a8c1::/33")
+
+	// The first two IPs in the /24 v4 ranges can be used for DoH to NextDNS.
+	//
+	// They're Anycast and usually okay, but NextDNS has some locations that
+	// don't do BGP and can get results for querying them over DoH to find the
+	// IPv4 address of "dns.mynextdns.io" and find an even better result.
+	//
+	// Note that the Tailscale DNS client does not do any of the "IP address
+	// linking" that NextDNS can do with its IPv4 addresses. These addresses
+	// are only used for DoH.
+	nextDNSv4RangeA = netip.MustParsePrefix("45.90.28.0/24")
+	nextDNSv4RangeB = netip.MustParsePrefix("45.90.30.0/24")
+	nextDNSv4One    = nextDNSv4RangeA.Addr()
+	nextDNSv4Two    = nextDNSv4RangeB.Addr()
+)
+
+// nextDNSv6Gen generates a NextDNS IPv6 address from the upper 8 bytes in the
+// provided ip and using id as the lowest 0-8 bytes.
+func nextDNSv6Gen(ip netip.Addr, id []byte) netip.Addr {
+	if len(id) > 8 {
+		return netip.Addr{}
+	}
+	a := ip.As16()
+	copy(a[16-len(id):], id)
+	return netip.AddrFrom16(a)
+}
+
+// IPIsDoHOnlyServer reports whether ip is a DNS server that should only use
+// DNS-over-HTTPS (not regular port 53 DNS).
+func IPIsDoHOnlyServer(ip netip.Addr) bool {
+	return nextDNSv6RangeA.Contains(ip) || nextDNSv6RangeB.Contains(ip) ||
+		nextDNSv4RangeA.Contains(ip) || nextDNSv4RangeB.Contains(ip)
+}

net/dns/publicdns/publicdns_test.go

@@ -6,20 +6,30 @@ package publicdns
 
 import (
 	"net/netip"
+	"reflect"
 	"testing"
 )
 
 func TestInit(t *testing.T) {
-	for baseKey, baseSet := range DoHIPsOfBase() {
+	for _, baseKey := range KnownDoHPrefixes() {
+		baseSet := DoHIPsOfBase(baseKey)
 		for _, addr := range baseSet {
-			if KnownDoH()[addr] != baseKey {
+			back, only, ok := DoHEndpointFromIP(addr)
-				t.Errorf("Expected %v to map to %s, got %s", addr, baseKey, KnownDoH()[addr])
+			if !ok {
+				t.Errorf("DoHEndpointFromIP(%v) not mapped back to %v", addr, baseKey)
+				continue
+			}
+			if only {
+				t.Errorf("unexpected DoH only bit set for %v", addr)
+			}
+			if back != baseKey {
+				t.Errorf("Expected %v to map to %s, got %s", addr, baseKey, back)
 			}
 		}
 	}
 }
 
-func TestDohV6(t *testing.T) {
+func TestDoHV6(t *testing.T) {
 	tests := []struct {
 		in      string
 		firstIP netip.Addr
@@ -38,3 +48,49 @@ func TestDohV6(t *testing.T) {
 		})
 	}
 }
+
+func TestDoHIPsOfBase(t *testing.T) {
+	ips := func(s ...string) (ret []netip.Addr) {
+		for _, ip := range s {
+			ret = append(ret, netip.MustParseAddr(ip))
+		}
+		return
+	}
+	tests := []struct {
+		base string
+		want []netip.Addr
+	}{
+		{
+			base: "https://cloudflare-dns.com/dns-query",
+			want: ips("1.1.1.1", "1.0.0.1", "2606:4700:4700::1111", "2606:4700:4700::1001"),
+		},
+		{
+			base: "https://dns.nextdns.io/",
+			want: ips(),
+		},
+		{
+			base: "https://dns.nextdns.io/ff",
+			want: ips(
+				"45.90.28.0",
+				"45.90.30.0",
+				"2a07:a8c0::ff",
+				"2a07:a8c1::ff",
+			),
+		},
+		{
+			base: "https://dns.nextdns.io/c3a884",
+			want: ips(
+				"45.90.28.0",
+				"45.90.30.0",
+				"2a07:a8c0::c3:a884",
+				"2a07:a8c1::c3:a884",
+			),
+		},
+	}
+	for _, tt := range tests {
+		got := DoHIPsOfBase(tt.base)
+		if !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("DoHIPsOfBase(%q) = %v; want %v", tt.base, got, tt.want)
+		}
+	}
+}

net/dns/resolver/doh_test.go

@@ -41,7 +41,8 @@ func TestDoH(t *testing.T) {
 	if !*testDoH {
 		t.Skip("skipping manual test without --test-doh flag")
 	}
-	if len(publicdns.KnownDoH()) == 0 {
+	prefixes := publicdns.KnownDoHPrefixes()
+	if len(prefixes) == 0 {
 		t.Fatal("no known DoH")
 	}
 
@@ -49,7 +50,7 @@ func TestDoH(t *testing.T) {
 		dohSem: make(chan struct{}, 10),
 	}
 
-	for urlBase := range publicdns.DoHIPsOfBase() {
+	for _, urlBase := range prefixes {
 		t.Run(urlBase, func(t *testing.T) {
 			c, ok := f.getKnownDoHClientForProvider(urlBase)
 			if !ok {
@@ -86,13 +87,15 @@ func TestDoH(t *testing.T) {
 }
 
 func TestDoHV6Fallback(t *testing.T) {
-	for ip, base := range publicdns.KnownDoH() {
+	for _, base := range publicdns.KnownDoHPrefixes() {
-		if ip.Is4() {
+		for _, ip := range publicdns.DoHIPsOfBase(base) {
-			ip6, ok := publicdns.DoHV6(base)
+			if ip.Is4() {
-			if !ok {
+				ip6, ok := publicdns.DoHV6(base)
-				t.Errorf("no v6 DoH known for %v", ip)
+				if !ok {
-			} else if !ip6.Is6() {
+					t.Errorf("no v6 DoH known for %v", ip)
-				t.Errorf("dohV6(%q) returned non-v6 address %v", base, ip6)
+				} else if !ip6.Is6() {
+					t.Errorf("dohV6(%q) returned non-v6 address %v", base, ip6)
+				}
 			}
 		}
 	}

net/dns/resolver/forwarder.go

@@ -259,18 +259,26 @@ func (f *forwarder) Close() error {
 func resolversWithDelays(resolvers []*dnstype.Resolver) []resolverAndDelay {
 	rr := make([]resolverAndDelay, 0, len(resolvers)+2)
 
+	type dohState uint8
+	const addedDoH = dohState(1)
+	const addedDoHAndDontAddUDP = dohState(2)
+
 	// Add the known DoH ones first, starting immediately.
-	didDoH := map[string]bool{}
+	didDoH := map[string]dohState{}
 	for _, r := range resolvers {
 		ipp, ok := r.IPPort()
 		if !ok {
 			continue
 		}
-		dohBase, ok := publicdns.KnownDoH()[ipp.Addr()]
+		dohBase, dohOnly, ok := publicdns.DoHEndpointFromIP(ipp.Addr())
-		if !ok || didDoH[dohBase] {
+		if !ok || didDoH[dohBase] != 0 {
 			continue
 		}
-		didDoH[dohBase] = true
+		if dohOnly {
+			didDoH[dohBase] = addedDoHAndDontAddUDP
+		} else {
+			didDoH[dohBase] = addedDoH
+		}
 		rr = append(rr, resolverAndDelay{name: &dnstype.Resolver{Addr: dohBase}})
 	}
 
@@ -289,8 +297,12 @@ func resolversWithDelays(resolvers []*dnstype.Resolver) []resolverAndDelay {
 		}
 		ip := ipp.Addr()
 		var startDelay time.Duration
-		if host, ok := publicdns.KnownDoH()[ip]; ok {
+		if host, _, ok := publicdns.DoHEndpointFromIP(ip); ok {
+			if didDoH[host] == addedDoHAndDontAddUDP {
+				continue
+			}
 			// We already did the DoH query early. These
+			// are for normal dns53 UDP queries.
 			startDelay = dohHeadStart
 			key := hostAndFam{host, uint8(ip.BitLen())}
 			if done[key] > 0 {
@@ -391,7 +403,7 @@ func (f *forwarder) getKnownDoHClientForProvider(urlBase string) (c *http.Client
 	if c, ok := f.dohClient[urlBase]; ok {
 		return c, true
 	}
-	allIPs := publicdns.DoHIPsOfBase()[urlBase]
+	allIPs := publicdns.DoHIPsOfBase(urlBase)
 	if len(allIPs) == 0 {
 		return nil, false
 	}
@@ -407,7 +419,7 @@ func (f *forwarder) getKnownDoHClientForProvider(urlBase string) (c *http.Client
 	c = &http.Client{
 		Transport: &http.Transport{
 			ForceAttemptHTTP2: true,
-			IdleConnTimeout: dohTransportTimeout,
+			IdleConnTimeout:   dohTransportTimeout,
 			DialContext: func(ctx context.Context, netw, addr string) (net.Conn, error) {
 				if !strings.HasPrefix(netw, "tcp") {
 					return nil, fmt.Errorf("unexpected network %q", netw)

net/dns/resolver/forwarder_test.go

@@ -79,6 +79,11 @@ func TestResolversWithDelays(t *testing.T) {
 			in:   q("9.9.9.9", "2620:fe::fe"),
 			want: o("https://dns.quad9.net/dns-query", "9.9.9.9+0.5s", "2620:fe::fe+0.5s"),
 		},
+		{
+			name: "nextdns-ipv6-expand",
+			in:   q("2a07:a8c0::c3:a884"),
+			want: o("https://dns.nextdns.io/c3a884"),
+		},
 	}
 
 	for _, tt := range tests {

tailcfg/tailcfg.go

@@ -78,7 +78,8 @@ type CapabilityVersion int
 //   - 39: 2022-08-15: clients can talk Noise over arbitrary HTTPS port
 //   - 40: 2022-08-22: added Node.KeySignature, PeersChangedPatch.KeySignature
 //   - 41: 2022-08-30: uses 100.100.100.100 for route-less ExtraRecords if global nameservers is set
-const CurrentCapabilityVersion CapabilityVersion = 41
+//   - 42: 2022-09-06: NextDNS DoH support; see https://github.com/tailscale/tailscale/pull/5556
+const CurrentCapabilityVersion CapabilityVersion = 42
 
 type StableID string