Add the presist storage for control routes

appc/appconnector.go

@@ -11,6 +11,7 @@ package appc
 
 import (
 	"context"
+	"fmt"
 	"net/netip"
 	"slices"
 	"strings"
@@ -67,6 +68,9 @@ type AppConnector struct {
 	// wildcards is the list of domain strings that match subdomains.
 	wildcards []string
 
+	// the in memory copy of all the routes that's advertised
+	routeInfo *routeinfo.RouteInfo
+
 	// queue provides ordering for update operations
 	queue execqueue.ExecQueue
 }
@@ -76,6 +80,7 @@ func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser) *AppConn
 	return &AppConnector{
 		logf:            logger.WithPrefix(logf, "appc: "),
 		routeAdvertiser: routeAdvertiser,
+		routeInfo:       routeinfo.NewRouteInfo(),
 	}
 }
 
@@ -150,12 +155,35 @@ func (e *AppConnector) updateRoutes(routes []netip.Prefix) {
 		return
 	}
 
+	routeInfo, err := e.routeAdvertiser.ReadRouteInfo()
+	if err != nil {
+		e.logf("failed to read routeInfo from store")
+	}
+	oldControl := routeInfo.Control
+	oldOtherRoutes := routeInfo.Routes(true, false, true)
+	fmt.Println("OldOtherRoutes: ", oldOtherRoutes)
+	var toRemove []netip.Prefix
+	for _, ipp := range oldControl {
+		if slices.Contains(routes, ipp) {
+			continue
+		}
+		// unadvertise the prefix if the prefix is not recorded from other source.
+		if !slices.Contains(oldOtherRoutes, ipp) {
+			toRemove = append(toRemove, ipp)
+		}
+	}
+
+	if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
+		e.logf("failed to unadvertise old routes: %v: %v", routes, err)
+	}
+	routeInfo.Control = routes
+	fmt.Println(toRemove)
 	if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil {
 		e.logf("failed to advertise routes: %v: %v", routes, err)
 		return
 	}
 
-	var toRemove []netip.Prefix
+	toRemove = toRemove[:0]
 
 nextRoute:
 	for _, r := range routes {
@@ -173,8 +201,9 @@ nextRoute:
 	if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
 		e.logf("failed to unadvertise routes: %v: %v", toRemove, err)
 	}
-
 	e.controlRoutes = routes
+	e.routeInfo = routeInfo
+	e.routeAdvertiser.StoreRouteInfo(e.routeInfo)
 }
 
 // Domains returns the currently configured domain list.

appc/appconnector_test.go

@@ -13,6 +13,7 @@ import (
 	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/appc/appctest"
+	"tailscale.com/appc/routeinfo"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 )
@@ -65,6 +66,10 @@ func TestUpdateRoutes(t *testing.T) {
 	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
 	a.updateRoutes(routes)
 
+	if !slices.Equal(a.routeInfo.Control, routes) {
+		t.Fatalf("got %v, want %v", a.routeInfo.Control, routes)
+	}
+
 	slices.SortFunc(rc.Routes(), prefixCompare)
 	rc.SetRoutes(slices.Compact(rc.Routes()))
 	slices.SortFunc(routes, prefixCompare)
@@ -81,6 +86,29 @@ func TestUpdateRoutes(t *testing.T) {
 	}
 }
 
+func TestUpdateRoutesNotUnadvertiseRoutesFromOtherSources(t *testing.T) {
+	rc := &appctest.RouteCollector{}
+	a := NewAppConnector(t.Logf, rc)
+	testRi := routeinfo.NewRouteInfo()
+	a.routeInfo.Local = []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
+	testRi.Local = append(testRi.Local, netip.MustParsePrefix("192.0.2.0/24"))
+	rc.StoreRouteInfo(testRi)
+
+	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
+	a.updateRoutes(routes)
+	if !slices.Equal(a.routeInfo.Control, routes) {
+		t.Fatalf("got %v, want %v", a.routeInfo.Control, routes)
+	}
+
+	routes2 := []netip.Prefix{netip.MustParsePrefix("192.0.0.1/32")}
+	a.updateRoutes(routes2)
+
+	wantRemoved := []netip.Prefix{}
+	if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
+		t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+	}
+}
+
 func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) {
 	rc := &appctest.RouteCollector{}
 	a := NewAppConnector(t.Logf, rc)

appc/appctest/appctest.go

@@ -14,6 +14,7 @@ import (
 type RouteCollector struct {
 	routes        []netip.Prefix
 	removedRoutes []netip.Prefix
+	routeInfo     *routeinfo.RouteInfo
 }
 
 func (rc *RouteCollector) AdvertiseRoute(pfx ...netip.Prefix) error {
@@ -35,11 +36,15 @@ func (rc *RouteCollector) UnadvertiseRoute(toRemove ...netip.Prefix) error {
 }
 
 func (rc *RouteCollector) StoreRouteInfo(ri *routeinfo.RouteInfo) error {
+	rc.routeInfo = ri
 	return nil
 }
 
 func (rc *RouteCollector) ReadRouteInfo() (*routeinfo.RouteInfo, error) {
-	return nil, nil
+	if rc.routeInfo == nil {
+		return routeinfo.NewRouteInfo(), nil
+	}
+	return rc.routeInfo, nil
 }
 
 // RemovedRoutes returns the list of routes that were removed.

appc/routeinfo/routeinfo.go

@@ -17,9 +17,46 @@ type RouteInfo struct {
 	Discovered map[string]*DatedRoutes
 }
 
+func NewRouteInfo() *RouteInfo {
+	discovered := make(map[string]*DatedRoutes)
+	return &RouteInfo{
+		Local:      []netip.Prefix{},
+		Control:    []netip.Prefix{},
+		Discovered: discovered,
+	}
+}
+
+// RouteInfo.Routes returns a slice containing all the routes stored from the wanted resources.
+func (ri *RouteInfo) Routes(local, control, discovered bool) []netip.Prefix {
+	var ret []netip.Prefix
+	if local {
+		ret = ri.Local
+	}
+	if control && len(ret) == 0 {
+		ret = ri.Control
+	} else if control {
+		ret = append(ret, ri.Control...)
+	}
+
+	if discovered {
+		for _, dr := range ri.Discovered {
+			ret = append(ret, dr.routesSlice()...)
+		}
+	}
+	return ret
+}
+
 type DatedRoutes struct {
 	// routes discovered for a domain, and when they were last seen in a dns query
 	Routes map[netip.Prefix]time.Time
 	// the time at which we last expired old routes
 	LastCleanup time.Time
 }
+
+func (dr *DatedRoutes) routesSlice() []netip.Prefix {
+	var routes []netip.Prefix
+	for k := range dr.Routes {
+		routes = append(routes, k)
+	}
+	return routes
+}