WIP: kernel accelerated tailscaled

This commit is contained in:
David Anderson 2021-08-30 14:30:06 -07:00
parent a35c3ba221
commit 9c0a1375eb
6 changed files with 428 additions and 18 deletions

View File

@ -19,6 +19,7 @@ import (
"net/http"
"net/http/pprof"
"os"
"os/exec"
"os/signal"
"runtime"
"runtime/debug"
@ -33,7 +34,6 @@ import (
"tailscale.com/logpolicy"
"tailscale.com/net/dns"
"tailscale.com/net/socks5/tssocks"
"tailscale.com/net/tstun"
"tailscale.com/paths"
"tailscale.com/types/flagtype"
"tailscale.com/types/logger"
@ -381,24 +381,33 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
}
useNetstack = name == "userspace-networking"
if !useNetstack {
dev, devName, err := tstun.New(logf, name)
if err != nil {
tstun.Diagnose(logf, name)
return nil, false, err
// dev, devName, err := tstun.New(logf, name)
// if err != nil {
// tstun.Diagnose(logf, name)
// return nil, false, err
// }
// conf.Tun = dev
// if strings.HasPrefix(name, "tap:") {
// conf.IsTAP = true
// e, err := wgengine.NewUserspaceEngine(logf, conf)
// return e, false, err
// }
// HACK
exec.Command("ip", "link", "del", "tailscale0").Run()
if err := exec.Command("ip", "link", "add", "tailscale0", "type", "wireguard").Run(); err != nil {
return nil, false, fmt.Errorf("create device: %v", err)
}
conf.Tun = dev
if strings.HasPrefix(name, "tap:") {
conf.IsTAP = true
e, err := wgengine.NewUserspaceEngine(logf, conf)
return e, false, err
if err := exec.Command("ip", "link", "set", "tailscale0", "up").Run(); err != nil {
return nil, false, fmt.Errorf("create device: %v", err)
}
r, err := router.New(logf, dev, linkMon)
r, err := router.New(logf, nil, linkMon)
if err != nil {
dev.Close()
//dev.Close()
return nil, false, err
}
d, err := dns.NewOSConfigurator(logf, devName)
d, err := dns.NewOSConfigurator(logf, "tailscale0")
if err != nil {
return nil, false, err
}
@ -408,7 +417,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
conf.Router = netstack.NewSubnetRouterWrapper(conf.Router)
}
}
e, err = wgengine.NewUserspaceEngine(logf, conf)
e, err = wgengine.NewKernelEngine(logf, conf)
if err != nil {
return nil, useNetstack, err
}

1
go.mod
View File

@ -45,6 +45,7 @@ require (
golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
golang.org/x/tools v0.1.2
golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20210803171230-4253848d036c // indirect
golang.zx2c4.com/wireguard/windows v0.3.16
honnef.co/go/tools v0.1.4
inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1

8
go.sum
View File

@ -421,6 +421,7 @@ github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY=
github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
@ -666,6 +667,7 @@ golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI=
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -733,6 +735,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210504132125-bbd867fde50d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@ -804,9 +807,11 @@ golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -891,8 +896,11 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.zx2c4.com/wireguard v0.0.0-20210427022245-097af6e1351b/go.mod h1:a057zjmoc00UN7gVkaJt2sXVK523kMJcogDTEvPIasg=
golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0 h1:qINUmOnDCCF7i14oomDDkGmlda7BSDTGfge77/aqdfk=
golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20210803171230-4253848d036c h1:ADNrRDI5NR23/TUCnEmlLZLt4u9DnZ2nwRkPrAcFvto=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20210803171230-4253848d036c/go.mod h1:+1XihzyZUBJcSc5WO9SwNA7v26puQwOEDwanaxfNXPQ=
golang.zx2c4.com/wireguard/windows v0.3.16 h1:S42i0kp3SFHZm1mMFTtiU3OnEQJ0GRVOVlMkBhSDTZI=
golang.zx2c4.com/wireguard/windows v0.3.16/go.mod h1:f80rkFY2CKQklps1GHE15k/M4Tq78aofbr1iQM5MTVY=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=

255
wgengine/kernel.go Normal file
View File

@ -0,0 +1,255 @@
// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package wgengine
import (
"errors"
"fmt"
"sync"
"golang.zx2c4.com/wireguard/wgctrl"
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
"inet.af/netaddr"
"tailscale.com/health"
"tailscale.com/ipn/ipnstate"
"tailscale.com/net/dns"
"tailscale.com/tailcfg"
"tailscale.com/types/key"
"tailscale.com/types/logger"
"tailscale.com/types/netmap"
"tailscale.com/types/wgkey"
"tailscale.com/wgengine/filter"
"tailscale.com/wgengine/kproxy"
"tailscale.com/wgengine/magicsock"
"tailscale.com/wgengine/monitor"
"tailscale.com/wgengine/router"
"tailscale.com/wgengine/wgcfg"
)
type kernelEngine struct {
logf logger.Logf
magicConn *magicsock.Conn
linkMon *monitor.Mon
linkMonOwned bool // whether we created linkMon (and thus need to close it)
router router.Router
dns *dns.Manager
confListenPort uint16 // original conf.ListenPort
wg *wgctrl.Client
proxy *kproxy.Proxy
proxyMap map[tailcfg.NodeKey]netaddr.IPPort
wgLock sync.Mutex
}
func NewKernelEngine(logf logger.Logf, conf Config) (_ Engine, reterr error) {
var closePool closeOnErrorPool
defer closePool.closeAllIfError(&reterr)
const tunName = "tailscale0" // TODO: plumb somehow for variable name
if conf.Tun != nil {
return nil, errors.New("can't use a tun interface in kernel mode")
}
if conf.Router == nil {
conf.Router = router.NewFake(logf)
}
if conf.DNS == nil {
d, err := dns.NewNoopManager()
if err != nil {
return nil, err
}
conf.DNS = d
}
e := &kernelEngine{
logf: logf,
router: conf.Router,
confListenPort: conf.ListenPort,
}
if conf.LinkMonitor != nil {
e.linkMon = conf.LinkMonitor
} else {
mon, err := monitor.New(logf)
if err != nil {
return nil, err
}
closePool.add(mon)
e.linkMon = mon
e.linkMonOwned = true
}
e.dns = dns.NewManager(logf, conf.DNS, e.linkMon, nil) // TODO: make a fwdLinkSelector
magicsockOpts := magicsock.Options{
Logf: logf,
Port: conf.ListenPort,
LinkMonitor: e.linkMon,
}
var err error
e.magicConn, err = magicsock.NewConn(magicsockOpts)
if err != nil {
return nil, fmt.Errorf("wgengine: %v", err)
}
closePool.add(e.magicConn)
e.magicConn.SetNetworkUp(true)
e.proxy, err = kproxy.New(e.magicConn)
if err != nil {
return nil, fmt.Errorf("proxy: %v", err)
}
e.wg, err = wgctrl.New()
if err != nil {
return nil, fmt.Errorf("wgctrl: %v", err)
}
closePool.add(e.wg)
err = e.wg.ConfigureDevice(tunName, wgtypes.Config{
PrivateKey: &wgtypes.Key{},
ReplacePeers: true,
Peers: []wgtypes.PeerConfig{},
})
if err != nil {
return nil, fmt.Errorf("wgctrl: initial config: %v", err)
}
if err := e.router.Up(); err != nil {
return nil, err
}
e.magicConn.Start()
return e, nil
}
func (e *kernelEngine) Reconfig(wcfg *wgcfg.Config, rcfg *router.Config, dcfg *dns.Config, dbg *tailcfg.Debug) error {
if rcfg == nil {
panic("rcfg must not be nil")
}
if dcfg == nil {
panic("dcfg must not be nil")
}
e.wgLock.Lock()
defer e.wgLock.Unlock()
peerSet := make(map[key.Public]struct{}, len(wcfg.Peers))
for _, p := range wcfg.Peers {
peerSet[key.Public(p.PublicKey)] = struct{}{}
}
if err := e.magicConn.SetPrivateKey(wgkey.Private(wcfg.PrivateKey)); err != nil {
e.logf("wgengine: Reconfig: SetPrivateKey: %v", err)
}
e.magicConn.UpdatePeers(peerSet)
e.magicConn.SetPreferredPort(e.confListenPort)
port := 4242
cfg := wgtypes.Config{
PrivateKey: (*wgtypes.Key)(&wcfg.PrivateKey),
ListenPort: &port,
ReplacePeers: true,
}
for _, p := range wcfg.Peers {
v := wgtypes.PeerConfig{
PublicKey: wgtypes.Key(p.PublicKey),
Endpoint: e.proxyMap[tailcfg.NodeKey(p.PublicKey)].UDPAddr(),
ReplaceAllowedIPs: true,
}
for _, pfx := range p.AllowedIPs {
v.AllowedIPs = append(v.AllowedIPs, *pfx.IPNet())
}
cfg.Peers = append(cfg.Peers, v)
}
if err := e.wg.ConfigureDevice("tailscale0", cfg); err != nil {
return fmt.Errorf("configuring kernel: %v", err)
}
err := e.router.Set(rcfg)
health.SetRouterHealth(err)
if err != nil {
return err
}
// TODO: set DNS, but it'll just break my machine right now, I
// mean look at the state of me.
return nil
}
func (e *kernelEngine) GetFilter() *filter.Filter { return nil }
func (e *kernelEngine) SetFilter(f *filter.Filter) {}
func (e *kernelEngine) SetStatusCallback(cb StatusCallback) {}
func (e *kernelEngine) GetLinkMonitor() *monitor.Mon { return e.linkMon }
func (e *kernelEngine) RequestStatus() {}
func (e *kernelEngine) Close() {}
func (e *kernelEngine) Wait() {}
func (e *kernelEngine) LinkChange(isExpensive bool) {}
func (e *kernelEngine) SetDERPMap(m *tailcfg.DERPMap) {
e.magicConn.SetDERPMap(m)
}
func (e *kernelEngine) SetNetworkMap(nm *netmap.NetworkMap) {
e.magicConn.SetNetworkMap(nm)
m, err := e.proxy.SetNetworkMap(nm)
if err != nil {
e.logf("MUCH SADNESS: %v", err)
}
e.wgLock.Lock()
defer e.wgLock.Unlock()
e.proxyMap = m
}
func (e *kernelEngine) AddNetworkMapCallback(cb NetworkMapCallback) (rm func()) { return func() {} }
func (e *kernelEngine) SetNetInfoCallback(cb NetInfoCallback) {}
func (e *kernelEngine) DiscoPublicKey() tailcfg.DiscoKey { return e.magicConn.DiscoPublicKey() }
func (e *kernelEngine) getStatus() (*Status, error) {
// Grab derpConns before acquiring wgLock to not violate lock ordering;
// the DERPs method acquires magicsock.Conn.mu.
// (See comment in userspaceEngine's declaration.)
derpConns := e.magicConn.DERPs()
return &Status{
LocalAddrs: []tailcfg.Endpoint{},
Peers: []ipnstate.PeerStatusLite{},
DERPs: derpConns,
}, nil
}
func (e *kernelEngine) UpdateStatus(sb *ipnstate.StatusBuilder) {
st, err := e.getStatus()
if err != nil {
e.logf("wgengine: getStatus: %v", err)
return
}
for _, ps := range st.Peers {
sb.AddPeer(key.Public(ps.NodeKey), &ipnstate.PeerStatus{
RxBytes: int64(ps.RxBytes),
TxBytes: int64(ps.TxBytes),
LastHandshake: ps.LastHandshake,
InEngine: true,
})
}
e.magicConn.UpdateStatus(sb)
}
func (e *kernelEngine) Ping(ip netaddr.IP, useTSMP bool, cb func(*ipnstate.PingResult)) {}
func (e *kernelEngine) RegisterIPPortIdentity(ipp netaddr.IPPort, ip netaddr.IP) {}
func (e *kernelEngine) UnregisterIPPortIdentity(ipp netaddr.IPPort) {}
func (e *kernelEngine) WhoIsIPPort(netaddr.IPPort) (netaddr.IP, bool) { return netaddr.IP{}, false }

136
wgengine/kproxy/proxy.go Normal file
View File

@ -0,0 +1,136 @@
// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package kproxy
import (
"encoding/json"
"net"
"sync"
"golang.zx2c4.com/wireguard/conn"
"inet.af/netaddr"
"tailscale.com/tailcfg"
"tailscale.com/types/netmap"
"tailscale.com/types/wgkey"
"tailscale.com/wgengine/magicsock"
"tailscale.com/wgengine/wgcfg"
)
type Proxy struct {
mu sync.RWMutex
conn *magicsock.Conn
byKey map[tailcfg.NodeKey]*pipe
byEndpoint map[conn.Endpoint]*pipe
}
func New(c *magicsock.Conn) (*Proxy, error) {
ret := &Proxy{
conn: c,
byEndpoint: map[conn.Endpoint]*pipe{},
byKey: map[tailcfg.NodeKey]*pipe{},
}
fns, _, err := c.Bind().Open(0)
if err != nil {
return nil, err
}
for _, fn := range fns {
go func(fn conn.ReceiveFunc) {
for {
var bs [1500]byte
n, ep, err := fn(bs[:])
if err != nil {
// Sadness.
continue
}
ret.mu.RLock()
pip, ok := ret.byEndpoint[ep]
ret.mu.RUnlock()
if ok {
if _, err := pip.proxy.Write(bs[:n]); err != nil {
_ = err // TODO
}
}
}
}(fn)
}
return ret, nil
}
var proxyListenIP = netaddr.MustParseIPPort("127.0.0.1:0")
var wgIP = netaddr.MustParseIPPort("127.0.0.1:4242")
func (p *Proxy) SetNetworkMap(nm *netmap.NetworkMap) (map[tailcfg.NodeKey]netaddr.IPPort, error) {
p.mu.Lock()
defer p.mu.Unlock()
ret := make(map[tailcfg.NodeKey]netaddr.IPPort, len(nm.Peers))
for _, peer := range nm.Peers {
if pip, ok := p.byKey[peer.Key]; ok {
ret[peer.Key] = pip.proxyAddr
} else {
wgEp := wgcfg.Endpoints{
PublicKey: wgkey.Key(peer.Key),
DiscoKey: peer.DiscoKey,
}
bs, err := json.Marshal(wgEp)
if err != nil {
return nil, err
}
ep, err := p.conn.ParseEndpoint(string(bs))
if err != nil {
return nil, err
}
conn, err := net.DialUDP("udp4", proxyListenIP.UDPAddr(), wgIP.UDPAddr())
if err != nil {
return nil, err
}
connAddr := netaddr.MustParseIPPort(conn.LocalAddr().String())
pip = &pipe{
ep: ep,
proxy: conn,
proxyAddr: connAddr,
}
go func() {
for {
var bs [1500]byte
n, ua, err := conn.ReadFromUDP(bs[:])
if err != nil {
return // TODO: more noise
}
ip, ok := netaddr.FromStdIP(ua.IP)
if !ok {
// ???
continue
}
if netaddr.IPPortFrom(ip, uint16(ua.Port)) != wgIP {
// Random noise that isn't kernel wg
continue
}
if err := p.conn.Send(bs[:n], ep); err != nil {
// Probably complain a bit
continue
}
}
}()
p.byKey[peer.Key] = pip
p.byEndpoint[ep] = pip
ret[peer.Key] = pip.proxyAddr
}
}
for key, pip := range p.byKey {
if _, ok := ret[key]; !ok {
pip.proxy.Close()
delete(p.byKey, key)
delete(p.byEndpoint, pip.ep)
}
}
return ret, nil
}
type pipe struct {
ep conn.Endpoint
proxy net.Conn
proxyAddr netaddr.IPPort
}

View File

@ -125,10 +125,11 @@ type linuxRouter struct {
}
func newUserspaceRouter(logf logger.Logf, tunDev tun.Device, linkMon *monitor.Mon) (Router, error) {
tunname, err := tunDev.Name()
if err != nil {
return nil, err
}
tunname := "tailscale0"
// , err := tunDev.Name()
// if err != nil {
// return nil, err
// }
ipt4, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
if err != nil {