appc: setting AdvertiseRoutes explicitly discards app connector routes

This fixes bugs where after using the cli to set AdvertiseRoutes users
were finding that they had to restart tailscaled before the app
connector would advertise previously learned routes again. And seems
more in line with user expectations.

Fixes #11006
Signed-off-by: Fran Bull <fran@tailscale.com>

appc/appconnector.go

@@ -36,6 +36,17 @@ type RouteAdvertiser interface {
 	UnadvertiseRoute(...netip.Prefix) error
 }
 
+// RouteInfo is a data structure used to persist the in memory state of an AppConnector
+// so that we can know, even after a restart, which routes came from ACLs and which were
+// learned from domains.
+type RouteInfo struct {
+	// routes from the 'routes' section of an app connector acl
+	Control []netip.Prefix
+	// routes discovered by observing dns lookups for configured domains
+	Domains   map[string][]netip.Addr
+	Wildcards []string
+}
+
 // AppConnector is an implementation of an AppConnector that performs
 // its function as a subsystem inside of a tailscale node. At the control plane
 // side App Connector routing is configured in terms of domains rather than IP
@@ -49,6 +60,17 @@ type AppConnector struct {
 	logf            logger.Logf
 	routeAdvertiser RouteAdvertiser
 
+	// Whether this tailscaled should persist routes. Storing RouteInfo enables the app connector
+	// to forget routes when appropriate and should make routes smaller. While we are verifying that
+	// writing the RouteInfo to StateStore is a good solution (and doesn't for example cause writes
+	// that are too frequent or too large) use a controlknob to manage this flag.
+	// shouldStoreRoutes doesn't need to be protected by a mutex as long as we only set it at struct
+	// creation.
+	shouldStoreRoutes bool
+
+	// Func with which we will persist routes if shouldStoreRoutes is true.
+	storeRoutesFunc func(*RouteInfo) error
+
 	// mu guards the fields that follow
 	mu sync.Mutex
 
@@ -67,11 +89,44 @@ type AppConnector struct {
 }
 
 // NewAppConnector creates a new AppConnector.
-func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser) *AppConnector {
-	return &AppConnector{
-		logf:            logger.WithPrefix(logf, "appc: "),
-		routeAdvertiser: routeAdvertiser,
+func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser, routeInfo *RouteInfo, storeRoutesFunc func(*RouteInfo) error, shouldStoreRoutes bool) *AppConnector {
+	ac := &AppConnector{
+		logf:              logger.WithPrefix(logf, "appc: "),
+		routeAdvertiser:   routeAdvertiser,
+		shouldStoreRoutes: shouldStoreRoutes,
+		storeRoutesFunc:   storeRoutesFunc,
 	}
+	if shouldStoreRoutes {
+		ac.domains = routeInfo.Domains
+		ac.wildcards = routeInfo.Wildcards
+		ac.controlRoutes = routeInfo.Control
+	}
+	return ac
+}
+
+// ShouldStoreRoutes returns true if the appconnector was created with the controlknob on
+// and is storing its discovered routes persistently.
+func (e *AppConnector) ShouldStoreRoutes() bool {
+	return e.shouldStoreRoutes
+}
+
+// storeRoutes takes the current state of the AppConnector and persists it
+func (e *AppConnector) storeRoutes() error {
+	return e.storeRoutesFunc(&RouteInfo{
+		Control:   e.controlRoutes,
+		Domains:   e.domains,
+		Wildcards: e.wildcards,
+	})
+}
+
+// ClearRoutes removes all route state from the AppConnector.
+func (e *AppConnector) ClearRoutes() error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.controlRoutes = nil
+	e.domains = nil
+	e.wildcards = nil
+	return e.storeRoutes()
 }
 
 // UpdateDomainsAndRoutes starts an asynchronous update of the configuration
@@ -125,10 +180,26 @@ func (e *AppConnector) updateDomains(domains []string) {
 		for _, wc := range e.wildcards {
 			if dnsname.HasSuffix(d, wc) {
 				e.domains[d] = addrs
+				delete(oldDomains, d)
 				break
 			}
 		}
 	}
+
+	// Everything left in oldDomains is a domain we're no longer tracking
+	// and if we are storing route info we can unadvertise the routes
+	if e.shouldStoreRoutes {
+		toRemove := []netip.Prefix{}
+		for _, addrs := range oldDomains {
+			for _, a := range addrs {
+				toRemove = append(toRemove, netip.PrefixFrom(a, a.BitLen()))
+			}
+		}
+		if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
+			e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", xmaps.Keys(oldDomains), toRemove, err)
+		}
+	}
+
 	e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards)
 }
 
@@ -152,6 +223,14 @@ func (e *AppConnector) updateRoutes(routes []netip.Prefix) {
 
 	var toRemove []netip.Prefix
 
+	// If we're storing routes and know e.controlRoutes is a good
+	// representation of what should be in AdvertisedRoutes we can stop
+	// advertising routes that used to be in e.controlRoutes but are not
+	// in routes.
+	if e.shouldStoreRoutes {
+		toRemove = routesWithout(e.controlRoutes, routes)
+	}
+
 nextRoute:
 	for _, r := range routes {
 		for _, addr := range e.domains {
@@ -170,6 +249,11 @@ nextRoute:
 	}
 
 	e.controlRoutes = routes
+	if e.shouldStoreRoutes {
+		if err := e.storeRoutes(); err != nil {
+			e.logf("failed to store route info: %v", err)
+		}
+	}
 }
 
 // Domains returns the currently configured domain list.
@@ -380,6 +464,11 @@ func (e *AppConnector) scheduleAdvertisement(domain string, routes ...netip.Pref
 				e.logf("[v2] advertised route for %v: %v", domain, addr)
 			}
 		}
+		if e.shouldStoreRoutes {
+			if err := e.storeRoutes(); err != nil {
+				e.logf("failed to store route info: %v", err)
+			}
+		}
 	})
 }
 
@@ -400,3 +489,43 @@ func (e *AppConnector) addDomainAddrLocked(domain string, addr netip.Addr) {
 func compareAddr(l, r netip.Addr) int {
 	return l.Compare(r)
 }
+
+func prefixCompare(a, b netip.Prefix) int {
+	if a.Addr().Compare(b.Addr()) == 0 {
+		return a.Bits() - b.Bits()
+	}
+	return a.Addr().Compare(b.Addr())
+}
+
+// routesWithout returns a without b where a and b
+// are unsorted slices of netip.Prefix
+func routesWithout(a, b []netip.Prefix) []netip.Prefix {
+	sa := a[:]
+	slices.SortFunc(sa, prefixCompare)
+	sb := b[:]
+	slices.SortFunc(sb, prefixCompare)
+	ai := 0
+	bi := 0
+	without := []netip.Prefix{}
+	for ai < len(sa) && bi < len(sb) {
+		if sa[ai] == sb[bi] {
+			ai++
+			bi++
+			continue
+		}
+		diff := prefixCompare(sa[ai], sb[bi])
+		if diff < 0 {
+			without = append(without, sa[ai])
+			ai++
+			continue
+		}
+		bi++
+	}
+	// if we're not at the end of sa all remaining items can be added to without
+	// if we're not at the end of sb it doesn't matter (those items are not in sa)
+	for ai < len(sa) {
+		without = append(without, sa[ai])
+		ai++
+	}
+	return without
+}

appc/appconnector_test.go

@@ -17,194 +17,208 @@ import (
 	"tailscale.com/util/must"
 )
 
+func fakeStoreRoutes(*RouteInfo) error { return nil }
+
 func TestUpdateDomains(t *testing.T) {
-	ctx := context.Background()
-	a := NewAppConnector(t.Logf, nil)
-	a.UpdateDomains([]string{"example.com"})
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		a := NewAppConnector(t.Logf, &appctest.RouteCollector{}, &RouteInfo{}, nil, shouldStore)
+		a.UpdateDomains([]string{"example.com"})
 
-	a.Wait(ctx)
-	if got, want := a.Domains().AsSlice(), []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.Wait(ctx)
+		if got, want := a.Domains().AsSlice(), []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	addr := netip.MustParseAddr("192.0.0.8")
-	a.domains["example.com"] = append(a.domains["example.com"], addr)
-	a.UpdateDomains([]string{"example.com"})
-	a.Wait(ctx)
+		addr := netip.MustParseAddr("192.0.0.8")
+		a.domains["example.com"] = append(a.domains["example.com"], addr)
+		a.UpdateDomains([]string{"example.com"})
+		a.Wait(ctx)
 
-	if got, want := a.domains["example.com"], []netip.Addr{addr}; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		if got, want := a.domains["example.com"], []netip.Addr{addr}; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// domains are explicitly downcased on set.
-	a.UpdateDomains([]string{"UP.EXAMPLE.COM"})
-	a.Wait(ctx)
-	if got, want := xmaps.Keys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
+		// domains are explicitly downcased on set.
+		a.UpdateDomains([]string{"UP.EXAMPLE.COM"})
+		a.Wait(ctx)
+		if got, want := xmaps.Keys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 	}
 }
 
 func TestUpdateRoutes(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	a.updateDomains([]string{"*.example.com"})
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes, shouldStore)
+		a.updateDomains([]string{"*.example.com"})
 
-	// This route should be collapsed into the range
-	a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
-	a.Wait(ctx)
+		// This route should be collapsed into the range
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
+		a.Wait(ctx)
 
-	if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
-		t.Fatalf("got %v, want %v", rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
-	}
+		if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
+			t.Fatalf("got %v, want %v", rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		}
 
-	// This route should not be collapsed or removed
-	a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
-	a.Wait(ctx)
+		// This route should not be collapsed or removed
+		a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
+		a.Wait(ctx)
 
-	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
-	a.updateRoutes(routes)
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
+		a.updateRoutes(routes)
 
-	slices.SortFunc(rc.Routes(), prefixCompare)
-	rc.SetRoutes(slices.Compact(rc.Routes()))
-	slices.SortFunc(routes, prefixCompare)
+		slices.SortFunc(rc.Routes(), prefixCompare)
+		rc.SetRoutes(slices.Compact(rc.Routes()))
+		slices.SortFunc(routes, prefixCompare)
 
-	// Ensure that the non-matching /32 is preserved, even though it's in the domains table.
-	if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
-		t.Errorf("added routes: got %v, want %v", rc.Routes(), routes)
-	}
+		// Ensure that the non-matching /32 is preserved, even though it's in the domains table.
+		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
+			t.Errorf("added routes: got %v, want %v", rc.Routes(), routes)
+		}
 
-	// Ensure that the contained /32 is removed, replaced by the /24.
-	wantRemoved := []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}
-	if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
-		t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		// Ensure that the contained /32 is removed, replaced by the /24.
+		wantRemoved := []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}
+		if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
+			t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		}
 	}
 }
 
 func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) {
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")})
-	rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
-	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
-	a.updateRoutes(routes)
+	for _, shouldStore := range []bool{false, true} {
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes, shouldStore)
+		mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")})
+		rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
+		a.updateRoutes(routes)
 
-	if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
-		t.Fatalf("got %v, want %v", rc.Routes(), routes)
+		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
+			t.Fatalf("got %v, want %v", rc.Routes(), routes)
+		}
 	}
 }
 
 func TestDomainRoutes(t *testing.T) {
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	a.updateDomains([]string{"example.com"})
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	a.Wait(context.Background())
+	for _, shouldStore := range []bool{false, true} {
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes, shouldStore)
+		a.updateDomains([]string{"example.com"})
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		a.Wait(context.Background())
 
-	want := map[string][]netip.Addr{
-		"example.com": {netip.MustParseAddr("192.0.0.8")},
-	}
+		want := map[string][]netip.Addr{
+			"example.com": {netip.MustParseAddr("192.0.0.8")},
+		}
 
-	if got := a.DomainRoutes(); !reflect.DeepEqual(got, want) {
-		t.Fatalf("DomainRoutes: got %v, want %v", got, want)
+		if got := a.DomainRoutes(); !reflect.DeepEqual(got, want) {
+			t.Fatalf("DomainRoutes: got %v, want %v", got, want)
+		}
 	}
 }
 
 func TestObserveDNSResponse(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes, shouldStore)
 
-	// a has no domains configured, so it should not advertise any routes
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a has no domains configured, so it should not advertise any routes
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
 
-	a.updateDomains([]string{"example.com"})
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"example.com"})
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// a CNAME record chain should result in a route being added if the chain
-	// matches a routed domain.
-	a.updateDomains([]string{"www.example.com", "example.com"})
-	a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
-	a.Wait(ctx)
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a CNAME record chain should result in a route being added if the chain
+		// matches a routed domain.
+		a.updateDomains([]string{"www.example.com", "example.com"})
+		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
+		a.Wait(ctx)
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// a CNAME record chain should result in a route being added if the chain
-	// even if only found in the middle of the chain
-	a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
-	a.Wait(ctx)
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a CNAME record chain should result in a route being added if the chain
+		// even if only found in the middle of the chain
+		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
+		a.Wait(ctx)
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
 
-	a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// don't re-advertise routes that have already been advertised
-	a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
-	a.Wait(ctx)
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
-	}
+		// don't re-advertise routes that have already been advertised
+		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		a.Wait(ctx)
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
+		}
 
-	// don't advertise addresses that are already in a control provided route
-	pfx := netip.MustParsePrefix("192.0.2.0/24")
-	a.updateRoutes([]netip.Prefix{pfx})
-	wantRoutes = append(wantRoutes, pfx)
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
-	a.Wait(ctx)
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
-	}
-	if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) {
-		t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"])
+		// don't advertise addresses that are already in a control provided route
+		pfx := netip.MustParsePrefix("192.0.2.0/24")
+		a.updateRoutes([]netip.Prefix{pfx})
+		wantRoutes = append(wantRoutes, pfx)
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
+		a.Wait(ctx)
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
+		}
+		if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) {
+			t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"])
+		}
 	}
 }
 
 func TestWildcardDomains(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes, shouldStore)
 
-	a.updateDomains([]string{"*.example.com"})
-	a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
-		t.Errorf("routes: got %v; want %v", got, want)
-	}
-	if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("wildcards: got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"*.example.com"})
+		a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
+			t.Errorf("routes: got %v; want %v", got, want)
+		}
+		if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("wildcards: got %v; want %v", got, want)
+		}
 
-	a.updateDomains([]string{"*.example.com", "example.com"})
-	if _, ok := a.domains["foo.example.com"]; !ok {
-		t.Errorf("expected foo.example.com to be preserved in domains due to wildcard")
-	}
-	if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("wildcards: got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"*.example.com", "example.com"})
+		if _, ok := a.domains["foo.example.com"]; !ok {
+			t.Errorf("expected foo.example.com to be preserved in domains due to wildcard")
+		}
+		if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("wildcards: got %v; want %v", got, want)
+		}
 
-	// There was an early regression where the wildcard domain was added repeatedly, this guards against that.
-	a.updateDomains([]string{"*.example.com", "example.com"})
-	if len(a.wildcards) != 1 {
-		t.Errorf("expected only one wildcard domain, got %v", a.wildcards)
+		// There was an early regression where the wildcard domain was added repeatedly, this guards against that.
+		a.updateDomains([]string{"*.example.com", "example.com"})
+		if len(a.wildcards) != 1 {
+			t.Errorf("expected only one wildcard domain, got %v", a.wildcards)
+		}
 	}
 }
 
@@ -304,9 +318,153 @@ func prefixEqual(a, b netip.Prefix) bool {
 	return a == b
 }
 
-func prefixCompare(a, b netip.Prefix) int {
-	if a.Addr().Compare(b.Addr()) == 0 {
-		return a.Bits() - b.Bits()
+func prefixes(in ...string) []netip.Prefix {
+	toRet := make([]netip.Prefix, len(in))
+	for i, s := range in {
+		toRet[i] = netip.MustParsePrefix(s)
 	}
-	return a.Addr().Compare(b.Addr())
+	return toRet
+}
+
+func TestUpdateRouteRouteRemoval(t *testing.T) {
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+
+		assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) {
+			if !slices.Equal(routes, rc.Routes()) {
+				t.Fatalf("%s: (shouldStore=%t) routes want %v, got %v", prefix, shouldStore, routes, rc.Routes())
+			}
+			if !slices.Equal(removedRoutes, rc.RemovedRoutes()) {
+				t.Fatalf("%s: (shouldStore=%t) removedRoutes want %v, got %v", prefix, shouldStore, removedRoutes, rc.RemovedRoutes())
+			}
+		}
+
+		a := NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes, shouldStore)
+		// nothing has yet been advertised
+		assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{}, prefixes("1.2.3.1/32", "1.2.3.2/32"))
+		a.Wait(ctx)
+		// the routes passed to UpdateDomainsAndRoutes have been advertised
+		assertRoutes("simple update", prefixes("1.2.3.1/32", "1.2.3.2/32"), []netip.Prefix{})
+
+		// one route the same, one different
+		a.UpdateDomainsAndRoutes([]string{}, prefixes("1.2.3.1/32", "1.2.3.3/32"))
+		a.Wait(ctx)
+		// old behavior: routes are not removed, resulting routes are both old and new
+		// (we have dupe 1.2.3.1 routes because the test RouteAdvertiser doesn't have the deduplication
+		// the real one does)
+		wantRoutes := prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.1/32", "1.2.3.3/32")
+		wantRemovedRoutes := []netip.Prefix{}
+		if shouldStore {
+			// new behavior: routes are removed, resulting routes are new only
+			wantRoutes = prefixes("1.2.3.1/32", "1.2.3.1/32", "1.2.3.3/32")
+			wantRemovedRoutes = prefixes("1.2.3.2/32")
+		}
+		assertRoutes("removal", wantRoutes, wantRemovedRoutes)
+	}
+}
+
+func TestUpdateDomainRouteRemoval(t *testing.T) {
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+
+		assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) {
+			if !slices.Equal(routes, rc.Routes()) {
+				t.Fatalf("%s: (shouldStore=%t) routes want %v, got %v", prefix, shouldStore, routes, rc.Routes())
+			}
+			if !slices.Equal(removedRoutes, rc.RemovedRoutes()) {
+				t.Fatalf("%s: (shouldStore=%t) removedRoutes want %v, got %v", prefix, shouldStore, removedRoutes, rc.RemovedRoutes())
+			}
+		}
+
+		a := NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes, shouldStore)
+		assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com", "b.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// adding domains doesn't immediately cause any routes to be advertised
+		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
+
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
+		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.3"))
+		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.4"))
+		a.Wait(ctx)
+		// observing dns responses causes routes to be advertised
+		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// old behavior, routes are not removed
+		wantRoutes := prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32")
+		wantRemovedRoutes := []netip.Prefix{}
+		if shouldStore {
+			// new behavior, routes are removed for b.example.com
+			wantRoutes = prefixes("1.2.3.1/32", "1.2.3.2/32")
+			wantRemovedRoutes = prefixes("1.2.3.3/32", "1.2.3.4/32")
+		}
+		assertRoutes("removal", wantRoutes, wantRemovedRoutes)
+	}
+}
+
+func TestUpdateWildcardRouteRemoval(t *testing.T) {
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+
+		assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) {
+			if !slices.Equal(routes, rc.Routes()) {
+				t.Fatalf("%s: (shouldStore=%t) routes want %v, got %v", prefix, shouldStore, routes, rc.Routes())
+			}
+			if !slices.Equal(removedRoutes, rc.RemovedRoutes()) {
+				t.Fatalf("%s: (shouldStore=%t) removedRoutes want %v, got %v", prefix, shouldStore, removedRoutes, rc.RemovedRoutes())
+			}
+		}
+
+		a := NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes, shouldStore)
+		assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com", "*.b.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// adding domains doesn't immediately cause any routes to be advertised
+		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
+
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
+		a.ObserveDNSResponse(dnsResponse("1.b.example.com.", "1.2.3.3"))
+		a.ObserveDNSResponse(dnsResponse("2.b.example.com.", "1.2.3.4"))
+		a.Wait(ctx)
+		// observing dns responses causes routes to be advertised
+		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// old behavior, routes are not removed
+		wantRoutes := prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32")
+		wantRemovedRoutes := []netip.Prefix{}
+		if shouldStore {
+			// new behavior, routes are removed for *.b.example.com
+			wantRoutes = prefixes("1.2.3.1/32", "1.2.3.2/32")
+			wantRemovedRoutes = prefixes("1.2.3.3/32", "1.2.3.4/32")
+		}
+		assertRoutes("removal", wantRoutes, wantRemovedRoutes)
+	}
+}
+
+func TestRoutesWithout(t *testing.T) {
+	assert := func(msg string, got, want []netip.Prefix) {
+		if !slices.Equal(want, got) {
+			t.Errorf("%s: want %v, got %v", msg, want, got)
+		}
+	}
+
+	assert("empty routes", routesWithout([]netip.Prefix{}, []netip.Prefix{}), []netip.Prefix{})
+	assert("a empty", routesWithout([]netip.Prefix{}, prefixes("1.1.1.1/32", "1.1.1.2/32")), []netip.Prefix{})
+	assert("b empty", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32"), []netip.Prefix{}), prefixes("1.1.1.1/32", "1.1.1.2/32"))
+	assert("no overlap", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32"), prefixes("1.1.1.3/32", "1.1.1.4/32")), prefixes("1.1.1.1/32", "1.1.1.2/32"))
+	assert("a has fewer", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32"), prefixes("1.1.1.1/32", "1.1.1.2/32", "1.1.1.3/32", "1.1.1.4/32")), []netip.Prefix{})
+	assert("a has more", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32", "1.1.1.3/32", "1.1.1.4/32"), prefixes("1.1.1.1/32", "1.1.1.3/32")), prefixes("1.1.1.2/32", "1.1.1.4/32"))
 }

control/controlknobs/controlknobs.go

@@ -72,6 +72,10 @@ type Knobs struct {
 	// ProbeUDPLifetime is whether the node should probe UDP path lifetime on
 	// the tail end of an active direct connection in magicsock.
 	ProbeUDPLifetime atomic.Bool
+
+	// AppCStoreRoutes is whether the node should store RouteInfo to StateStore
+	// if it's an app connector.
+	AppCStoreRoutes atomic.Bool
 }
 
 // UpdateFromNodeAttributes updates k (if non-nil) based on the provided self
@@ -96,6 +100,7 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) {
 		forceNfTables                 = has(tailcfg.NodeAttrLinuxMustUseNfTables)
 		seamlessKeyRenewal            = has(tailcfg.NodeAttrSeamlessKeyRenewal)
 		probeUDPLifetime              = has(tailcfg.NodeAttrProbeUDPLifetime)
+		appCStoreRoutes               = has(tailcfg.NodeAttrStoreAppCRoutes)
 	)
 
 	if has(tailcfg.NodeAttrOneCGNATEnable) {
@@ -118,6 +123,7 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) {
 	k.LinuxForceNfTables.Store(forceNfTables)
 	k.SeamlessKeyRenewal.Store(seamlessKeyRenewal)
 	k.ProbeUDPLifetime.Store(probeUDPLifetime)
+	k.AppCStoreRoutes.Store(appCStoreRoutes)
 }
 
 // AsDebugJSON returns k as something that can be marshalled with json.Marshal
@@ -141,5 +147,6 @@ func (k *Knobs) AsDebugJSON() map[string]any {
 		"LinuxForceNfTables":            k.LinuxForceNfTables.Load(),
 		"SeamlessKeyRenewal":            k.SeamlessKeyRenewal.Load(),
 		"ProbeUDPLifetime":              k.ProbeUDPLifetime.Load(),
+		"AppCStoreRoutes":               k.AppCStoreRoutes.Load(),
 	}
 }

ipn/ipnlocal/local.go

@@ -3114,6 +3114,19 @@ func (b *LocalBackend) SetUseExitNodeEnabled(v bool) (ipn.PrefsView, error) {
 	return b.editPrefsLockedOnEntry(mp, unlock)
 }
 
+// MaybeClearAppConnector clears the routes from any AppConnector if
+// AdvertiseRoutes has been set in the MaskedPrefs.
+func (b *LocalBackend) MaybeClearAppConnector(mp *ipn.MaskedPrefs) error {
+	var err error
+	if b.appConnector != nil && mp.AdvertiseRoutesSet {
+		err = b.appConnector.ClearRoutes()
+		if err != nil {
+			b.logf("appc: clear routes error: %v", err)
+		}
+	}
+	return err
+}
+
 func (b *LocalBackend) EditPrefs(mp *ipn.MaskedPrefs) (ipn.PrefsView, error) {
 	if mp.SetsInternal() {
 		return ipn.PrefsView{}, errors.New("can't set Internal fields")
@@ -3492,8 +3505,19 @@ func (b *LocalBackend) reconfigAppConnectorLocked(nm *netmap.NetworkMap, prefs i
 		return
 	}
 
-	if b.appConnector == nil {
-		b.appConnector = appc.NewAppConnector(b.logf, b)
+	shouldAppCStoreRoutes := b.ControlKnobs().AppCStoreRoutes.Load()
+	if b.appConnector == nil || b.appConnector.ShouldStoreRoutes() != shouldAppCStoreRoutes {
+		ri := &appc.RouteInfo{}
+		if shouldAppCStoreRoutes {
+			var err error
+			ri, err = b.readRouteInfoLocked()
+			if err != nil {
+				if err != ipn.ErrStateNotExist {
+					b.logf("Unsuccessful Read RouteInfo: ", err)
+				}
+			}
+		}
+		b.appConnector = appc.NewAppConnector(b.logf, b, ri, b.storeRouteInfo, shouldAppCStoreRoutes)
 	}
 	if nm == nil {
 		return
@@ -6186,6 +6210,43 @@ func (b *LocalBackend) UnadvertiseRoute(toRemove ...netip.Prefix) error {
 	return err
 }
 
+// namespace a key with the profile manager's current profile key, if any
+func namespaceKeyForCurrentProfile(pm *profileManager, key ipn.StateKey) ipn.StateKey {
+	return pm.CurrentProfile().Key + "||" + key
+}
+
+const routeInfoStateStoreKey ipn.StateKey = "_routeInfo"
+
+func (b *LocalBackend) storeRouteInfo(ri *appc.RouteInfo) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.pm.CurrentProfile().ID == "" {
+		return nil
+	}
+	key := namespaceKeyForCurrentProfile(b.pm, routeInfoStateStoreKey)
+	bs, err := json.Marshal(ri)
+	if err != nil {
+		return err
+	}
+	return b.pm.WriteState(key, bs)
+}
+
+func (b *LocalBackend) readRouteInfoLocked() (*appc.RouteInfo, error) {
+	if b.pm.CurrentProfile().ID == "" {
+		return &appc.RouteInfo{}, nil
+	}
+	key := namespaceKeyForCurrentProfile(b.pm, routeInfoStateStoreKey)
+	bs, err := b.pm.Store().ReadState(key)
+	ri := &appc.RouteInfo{}
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(bs, ri); err != nil {
+		return nil, err
+	}
+	return ri, nil
+}
+
 // seamlessRenewalEnabled reports whether seamless key renewals are enabled
 // (i.e. we saw our self node with the SeamlessKeyRenewal attr in a netmap).
 // This enables beta functionality of renewing node keys without breaking

ipn/ipnlocal/local_test.go

@@ -54,6 +54,8 @@ import (
 	"tailscale.com/wgengine/wgcfg"
 )
 
+func fakeStoreRoutes(*appc.RouteInfo) error { return nil }
+
 func inRemove(ip netip.Addr) bool {
 	for _, pfx := range removeFromDefaultRoute {
 		if pfx.Contains(ip) {
@@ -1290,13 +1292,15 @@ func TestDNSConfigForNetmapForExitNodeConfigs(t *testing.T) {
 }
 
 func TestOfferingAppConnector(t *testing.T) {
-	b := newTestBackend(t)
-	if b.OfferingAppConnector() {
-		t.Fatal("unexpected offering app connector")
-	}
-	b.appConnector = appc.NewAppConnector(t.Logf, nil)
-	if !b.OfferingAppConnector() {
-		t.Fatal("unexpected not offering app connector")
+	for _, shouldStore := range []bool{false, true} {
+		b := newTestBackend(t)
+		if b.OfferingAppConnector() {
+			t.Fatal("unexpected offering app connector")
+		}
+		b.appConnector = appc.NewAppConnector(t.Logf, nil, &appc.RouteInfo{}, nil, shouldStore)
+		if !b.OfferingAppConnector() {
+			t.Fatal("unexpected not offering app connector")
+		}
 	}
 }
 
@@ -1341,21 +1345,23 @@ func TestRouterAdvertiserIgnoresContainedRoutes(t *testing.T) {
 }
 
 func TestObserveDNSResponse(t *testing.T) {
-	b := newTestBackend(t)
+	for _, shouldStore := range []bool{false, true} {
+		b := newTestBackend(t)
 
-	// ensure no error when no app connector is configured
-	b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		// ensure no error when no app connector is configured
+		b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
 
-	rc := &appctest.RouteCollector{}
-	b.appConnector = appc.NewAppConnector(t.Logf, rc)
-	b.appConnector.UpdateDomains([]string{"example.com"})
-	b.appConnector.Wait(context.Background())
+		rc := &appctest.RouteCollector{}
+		b.appConnector = appc.NewAppConnector(t.Logf, rc, &appc.RouteInfo{}, fakeStoreRoutes, shouldStore)
+		b.appConnector.UpdateDomains([]string{"example.com"})
+		b.appConnector.Wait(context.Background())
 
-	b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	b.appConnector.Wait(context.Background())
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Fatalf("got routes %v, want %v", rc.Routes(), wantRoutes)
+		b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		b.appConnector.Wait(context.Background())
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Fatalf("got routes %v, want %v", rc.Routes(), wantRoutes)
+		}
 	}
 }
 
@@ -3439,3 +3445,66 @@ func TestEnableAutoUpdates(t *testing.T) {
 		t.Fatalf("disabling auto-updates: got error: %v", err)
 	}
 }
+
+func TestReadWriteRouteInfo(t *testing.T) {
+	// set up a backend with more than one profile
+	b := newTestBackend(t)
+	prof1 := ipn.LoginProfile{ID: "id1", Key: "key1"}
+	prof2 := ipn.LoginProfile{ID: "id2", Key: "key2"}
+	b.pm.knownProfiles["id1"] = &prof1
+	b.pm.knownProfiles["id2"] = &prof2
+	b.pm.currentProfile = &prof1
+
+	// set up routeInfo
+	ri1 := &appc.RouteInfo{}
+	ri1.Wildcards = []string{"1"}
+
+	ri2 := &appc.RouteInfo{}
+	ri2.Wildcards = []string{"2"}
+
+	// read before write
+	readRi, err := b.readRouteInfoLocked()
+	if readRi != nil {
+		t.Fatalf("read before writing: want nil, got %v", readRi)
+	}
+	if err != ipn.ErrStateNotExist {
+		t.Fatalf("read before writing: want %v, got %v", ipn.ErrStateNotExist, err)
+	}
+
+	// write the first routeInfo
+	if err := b.storeRouteInfo(ri1); err != nil {
+		t.Fatal(err)
+	}
+
+	// write the other routeInfo as the other profile
+	if err := b.pm.SwitchProfile("id2"); err != nil {
+		t.Fatal(err)
+	}
+	if err := b.storeRouteInfo(ri2); err != nil {
+		t.Fatal(err)
+	}
+
+	// read the routeInfo of the first profile
+	if err := b.pm.SwitchProfile("id1"); err != nil {
+		t.Fatal(err)
+	}
+	readRi, err = b.readRouteInfoLocked()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Equal(readRi.Wildcards, ri1.Wildcards) {
+		t.Fatalf("read prof1 routeInfo wildcards:  want %v, got %v", ri1.Wildcards, readRi.Wildcards)
+	}
+
+	// read the routeInfo of the second profile
+	if err := b.pm.SwitchProfile("id2"); err != nil {
+		t.Fatal(err)
+	}
+	readRi, err = b.readRouteInfoLocked()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Equal(readRi.Wildcards, ri2.Wildcards) {
+		t.Fatalf("read prof2 routeInfo wildcards:  want %v, got %v", ri2.Wildcards, readRi.Wildcards)
+	}
+}

ipn/ipnlocal/peerapi_test.go

@@ -687,185 +687,191 @@ func TestPeerAPIReplyToDNSQueries(t *testing.T) {
 }
 
 func TestPeerAPIPrettyReplyCNAME(t *testing.T) {
-	var h peerAPIHandler
-	h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
+	for _, shouldStore := range []bool{false, true} {
+		var h peerAPIHandler
+		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
-	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
-	pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
-	h.ps = &peerAPIServer{
-		b: &LocalBackend{
-			e:     eng,
-			pm:    pm,
-			store: pm.Store(),
-			// configure as an app connector just to enable the API.
-			appConnector: appc.NewAppConnector(t.Logf, &appctest.RouteCollector{}),
-		},
-	}
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
+		pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
+		h.ps = &peerAPIServer{
+			b: &LocalBackend{
+				e:     eng,
+				pm:    pm,
+				store: pm.Store(),
+				// configure as an app connector just to enable the API.
+				appConnector: appc.NewAppConnector(t.Logf, &appctest.RouteCollector{}, &appc.RouteInfo{}, nil, shouldStore),
+			},
+		}
 
-	h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
-		b.CNAMEResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("www.example.com."),
-				Type:  dnsmessage.TypeCNAME,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.CNAMEResource{
-				CNAME: dnsmessage.MustNewName("example.com."),
-			},
-		)
-		b.AResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("example.com."),
-				Type:  dnsmessage.TypeA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.AResource{
-				A: [4]byte{192, 0, 0, 8},
-			},
-		)
-	}}
-	f := filter.NewAllowAllForTest(logger.Discard)
-	h.ps.b.setFilter(f)
+		h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
+			b.CNAMEResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("www.example.com."),
+					Type:  dnsmessage.TypeCNAME,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.CNAMEResource{
+					CNAME: dnsmessage.MustNewName("example.com."),
+				},
+			)
+			b.AResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AResource{
+					A: [4]byte{192, 0, 0, 8},
+				},
+			)
+		}}
+		f := filter.NewAllowAllForTest(logger.Discard)
+		h.ps.b.setFilter(f)
 
-	if !h.replyToDNSQueries() {
-		t.Errorf("unexpectedly deny; wanted to be a DNS server")
-	}
+		if !h.replyToDNSQueries() {
+			t.Errorf("unexpectedly deny; wanted to be a DNS server")
+		}
 
-	w := httptest.NewRecorder()
-	h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
-	if w.Code != http.StatusOK {
-		t.Errorf("unexpected status code: %v", w.Code)
-	}
-	var addrs []string
-	json.NewDecoder(w.Body).Decode(&addrs)
-	if len(addrs) == 0 {
-		t.Fatalf("no addresses returned")
-	}
-	for _, addr := range addrs {
-		netip.MustParseAddr(addr)
+		w := httptest.NewRecorder()
+		h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
+		if w.Code != http.StatusOK {
+			t.Errorf("unexpected status code: %v", w.Code)
+		}
+		var addrs []string
+		json.NewDecoder(w.Body).Decode(&addrs)
+		if len(addrs) == 0 {
+			t.Fatalf("no addresses returned")
+		}
+		for _, addr := range addrs {
+			netip.MustParseAddr(addr)
+		}
 	}
 }
 
 func TestPeerAPIReplyToDNSQueriesAreObserved(t *testing.T) {
-	ctx := context.Background()
-	var h peerAPIHandler
-	h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		var h peerAPIHandler
+		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
-	rc := &appctest.RouteCollector{}
-	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
-	pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
-	h.ps = &peerAPIServer{
-		b: &LocalBackend{
-			e:            eng,
-			pm:           pm,
-			store:        pm.Store(),
-			appConnector: appc.NewAppConnector(t.Logf, rc),
-		},
-	}
-	h.ps.b.appConnector.UpdateDomains([]string{"example.com"})
-	h.ps.b.appConnector.Wait(ctx)
-
-	h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
-		b.AResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("example.com."),
-				Type:  dnsmessage.TypeA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
+		rc := &appctest.RouteCollector{}
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
+		pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
+		h.ps = &peerAPIServer{
+			b: &LocalBackend{
+				e:            eng,
+				pm:           pm,
+				store:        pm.Store(),
+				appConnector: appc.NewAppConnector(t.Logf, rc, &appc.RouteInfo{}, fakeStoreRoutes, shouldStore),
 			},
-			dnsmessage.AResource{
-				A: [4]byte{192, 0, 0, 8},
-			},
-		)
-	}}
-	f := filter.NewAllowAllForTest(logger.Discard)
-	h.ps.b.setFilter(f)
+		}
+		h.ps.b.appConnector.UpdateDomains([]string{"example.com"})
+		h.ps.b.appConnector.Wait(ctx)
 
-	if !h.ps.b.OfferingAppConnector() {
-		t.Fatal("expecting to be offering app connector")
-	}
-	if !h.replyToDNSQueries() {
-		t.Errorf("unexpectedly deny; wanted to be a DNS server")
-	}
+		h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
+			b.AResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AResource{
+					A: [4]byte{192, 0, 0, 8},
+				},
+			)
+		}}
+		f := filter.NewAllowAllForTest(logger.Discard)
+		h.ps.b.setFilter(f)
 
-	w := httptest.NewRecorder()
-	h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=example.com.", nil))
-	if w.Code != http.StatusOK {
-		t.Errorf("unexpected status code: %v", w.Code)
-	}
-	h.ps.b.appConnector.Wait(ctx)
+		if !h.ps.b.OfferingAppConnector() {
+			t.Fatal("expecting to be offering app connector")
+		}
+		if !h.replyToDNSQueries() {
+			t.Errorf("unexpectedly deny; wanted to be a DNS server")
+		}
 
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		w := httptest.NewRecorder()
+		h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=example.com.", nil))
+		if w.Code != http.StatusOK {
+			t.Errorf("unexpected status code: %v", w.Code)
+		}
+		h.ps.b.appConnector.Wait(ctx)
+
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		}
 	}
 }
 
 func TestPeerAPIReplyToDNSQueriesAreObservedWithCNAMEFlattening(t *testing.T) {
-	ctx := context.Background()
-	var h peerAPIHandler
-	h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		var h peerAPIHandler
+		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
-	rc := &appctest.RouteCollector{}
-	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
-	pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
-	h.ps = &peerAPIServer{
-		b: &LocalBackend{
-			e:            eng,
-			pm:           pm,
-			store:        pm.Store(),
-			appConnector: appc.NewAppConnector(t.Logf, rc),
-		},
-	}
-	h.ps.b.appConnector.UpdateDomains([]string{"www.example.com"})
-	h.ps.b.appConnector.Wait(ctx)
-
-	h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
-		b.CNAMEResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("www.example.com."),
-				Type:  dnsmessage.TypeCNAME,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
+		rc := &appctest.RouteCollector{}
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
+		pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
+		h.ps = &peerAPIServer{
+			b: &LocalBackend{
+				e:            eng,
+				pm:           pm,
+				store:        pm.Store(),
+				appConnector: appc.NewAppConnector(t.Logf, rc, &appc.RouteInfo{}, fakeStoreRoutes, shouldStore),
 			},
-			dnsmessage.CNAMEResource{
-				CNAME: dnsmessage.MustNewName("example.com."),
-			},
-		)
-		b.AResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("example.com."),
-				Type:  dnsmessage.TypeA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.AResource{
-				A: [4]byte{192, 0, 0, 8},
-			},
-		)
-	}}
-	f := filter.NewAllowAllForTest(logger.Discard)
-	h.ps.b.setFilter(f)
+		}
+		h.ps.b.appConnector.UpdateDomains([]string{"www.example.com"})
+		h.ps.b.appConnector.Wait(ctx)
 
-	if !h.ps.b.OfferingAppConnector() {
-		t.Fatal("expecting to be offering app connector")
-	}
-	if !h.replyToDNSQueries() {
-		t.Errorf("unexpectedly deny; wanted to be a DNS server")
-	}
+		h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
+			b.CNAMEResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("www.example.com."),
+					Type:  dnsmessage.TypeCNAME,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.CNAMEResource{
+					CNAME: dnsmessage.MustNewName("example.com."),
+				},
+			)
+			b.AResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AResource{
+					A: [4]byte{192, 0, 0, 8},
+				},
+			)
+		}}
+		f := filter.NewAllowAllForTest(logger.Discard)
+		h.ps.b.setFilter(f)
 
-	w := httptest.NewRecorder()
-	h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
-	if w.Code != http.StatusOK {
-		t.Errorf("unexpected status code: %v", w.Code)
-	}
-	h.ps.b.appConnector.Wait(ctx)
+		if !h.ps.b.OfferingAppConnector() {
+			t.Fatal("expecting to be offering app connector")
+		}
+		if !h.replyToDNSQueries() {
+			t.Errorf("unexpectedly deny; wanted to be a DNS server")
+		}
 
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		w := httptest.NewRecorder()
+		h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
+		if w.Code != http.StatusOK {
+			t.Errorf("unexpected status code: %v", w.Code)
+		}
+		h.ps.b.appConnector.Wait(ctx)
+
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		}
 	}
 }
 

ipn/localapi/localapi.go

@@ -1368,6 +1368,12 @@ func (h *Handler) servePrefs(w http.ResponseWriter, r *http.Request) {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
+		if err := h.b.MaybeClearAppConnector(mp); err != nil {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusInternalServerError)
+			json.NewEncoder(w).Encode(resJSON{Error: err.Error()})
+			return
+		}
 		var err error
 		prefs, err = h.b.EditPrefs(mp)
 		if err != nil {

tailcfg/tailcfg.go

@@ -2245,6 +2245,12 @@ const (
 
 	// NodeAttrLogExitFlows enables exit node destinations in network flow logs.
 	NodeAttrLogExitFlows NodeCapability = "log-exit-flows"
+
+	// NodeAttrExitDstNetworkFlowLog enables exit node destinations in network flow logs.
+	NodeAttrExitDstNetworkFlowLog NodeCapability = "exit-dst-network-flow-log"
+
+	// NodeAttrStoreAppCRoutes enables storing app connector routes persistently.
+	NodeAttrStoreAppCRoutes NodeCapability = "store-appc-routes"
 )
 
 // SetDNSRequest is a request to add a DNS record.