tsweb: move varz handler(s) into separate modules

This splits Prometheus metric handlers exposed by tsweb into two
modules:
- `varz.Handler` exposes Prometheus metrics generated by our expvar
  converter;
- `promvarz.Handler` combines our expvar-converted metrics and native
  Prometheus metrics.

By default, tsweb will use the promvarz handler, however users can keep
using only the expvar converter. Specifically, `tailscaled` now uses
`varz.Handler` explicitly, which avoids a dependency on the
(heavyweight) Prometheus client.

Updates https://github.com/tailscale/corp/issues/10205

Signed-off-by: Anton Tolchanov <anton@tailscale.com>
This commit is contained in:
Anton Tolchanov 2023-04-10 11:28:16 +01:00 committed by Anton Tolchanov
parent c153e6ae2f
commit 8546ff98fb
15 changed files with 850 additions and 808 deletions

View File

@ -24,7 +24,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
L 💣 github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+
L 💣 github.com/mdlayher/socket from github.com/mdlayher/netlink
💣 github.com/mitchellh/go-ps from tailscale.com/safesocket
💣 github.com/prometheus/client_golang/prometheus from tailscale.com/tsweb
💣 github.com/prometheus/client_golang/prometheus from tailscale.com/tsweb/promvarz
github.com/prometheus/client_golang/prometheus/internal from github.com/prometheus/client_golang/prometheus
github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+
github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+
@ -104,6 +104,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
💣 tailscale.com/tstime/mono from tailscale.com/tstime/rate
tailscale.com/tstime/rate from tailscale.com/wgengine/filter+
tailscale.com/tsweb from tailscale.com/cmd/derper
tailscale.com/tsweb/promvarz from tailscale.com/tsweb
tailscale.com/tsweb/varz from tailscale.com/tsweb+
tailscale.com/types/dnstype from tailscale.com/tailcfg
tailscale.com/types/empty from tailscale.com/ipn
tailscale.com/types/ipproto from tailscale.com/net/flowtrack+
@ -232,7 +234,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
net/http from expvar+
net/http/httptrace from net/http+
net/http/internal from net/http
net/http/pprof from tailscale.com/tsweb
net/http/pprof from tailscale.com/tsweb+
net/netip from go4.org/netipx+
net/textproto from golang.org/x/net/http/httpguts+
net/url from crypto/x509+

View File

@ -69,8 +69,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
L github.com/aws/smithy-go/transport/http from github.com/aws/aws-sdk-go-v2/aws/middleware+
L github.com/aws/smithy-go/transport/http/internal/io from github.com/aws/smithy-go/transport/http
L github.com/aws/smithy-go/waiter from github.com/aws/aws-sdk-go-v2/service/ssm
github.com/beorn7/perks/quantile from github.com/prometheus/client_golang/prometheus
💣 github.com/cespare/xxhash/v2 from github.com/prometheus/client_golang/prometheus
L github.com/coreos/go-iptables/iptables from tailscale.com/wgengine/router
LD 💣 github.com/creack/pty from tailscale.com/ssh/tailssh
W 💣 github.com/dblohm7/wingoes from github.com/dblohm7/wingoes/com
@ -80,8 +78,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
W 💣 github.com/go-ole/go-ole/oleutil from tailscale.com/wgengine/winnet
L 💣 github.com/godbus/dbus/v5 from tailscale.com/net/dns+
github.com/golang/groupcache/lru from tailscale.com/net/dnscache
github.com/golang/protobuf/proto from github.com/matttproud/golang_protobuf_extensions/pbutil+
github.com/golang/protobuf/ptypes/timestamp from github.com/prometheus/client_model/go
github.com/google/btree from gvisor.dev/gvisor/pkg/tcpip/header+
github.com/hdevalence/ed25519consensus from tailscale.com/tka
L 💣 github.com/illarion/gonotify from tailscale.com/net/dns
@ -103,7 +99,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd
github.com/kortschak/wol from tailscale.com/ipn/ipnlocal
LD github.com/kr/fs from github.com/pkg/sftp
github.com/matttproud/golang_protobuf_extensions/pbutil from github.com/prometheus/common/expfmt
L github.com/mdlayher/genetlink from tailscale.com/net/tstun
L 💣 github.com/mdlayher/netlink from github.com/jsimonetti/rtnetlink+
L 💣 github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+
@ -113,15 +108,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
W github.com/pkg/errors from github.com/tailscale/certstore
LD github.com/pkg/sftp from tailscale.com/ssh/tailssh
LD github.com/pkg/sftp/internal/encoding/ssh/filexfer from github.com/pkg/sftp
💣 github.com/prometheus/client_golang/prometheus from tailscale.com/tsweb
github.com/prometheus/client_golang/prometheus/internal from github.com/prometheus/client_golang/prometheus
github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+
github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+
github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt
github.com/prometheus/common/model from github.com/prometheus/client_golang/prometheus+
LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus
LD github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs
LD github.com/prometheus/procfs/internal/util from github.com/prometheus/procfs
W 💣 github.com/tailscale/certstore from tailscale.com/control/controlclient
LD github.com/tailscale/golang-x-crypto/chacha20 from github.com/tailscale/golang-x-crypto/ssh
LD 💣 github.com/tailscale/golang-x-crypto/internal/subtle from github.com/tailscale/golang-x-crypto/chacha20
@ -155,34 +141,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
go4.org/netipx from tailscale.com/ipn/ipnlocal+
W 💣 golang.zx2c4.com/wintun from github.com/tailscale/wireguard-go/tun+
W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg from tailscale.com/net/dns+
google.golang.org/protobuf/encoding/prototext from github.com/golang/protobuf/proto+
google.golang.org/protobuf/encoding/protowire from github.com/golang/protobuf/proto+
google.golang.org/protobuf/internal/descfmt from google.golang.org/protobuf/internal/filedesc
google.golang.org/protobuf/internal/descopts from google.golang.org/protobuf/internal/filedesc+
google.golang.org/protobuf/internal/detrand from google.golang.org/protobuf/internal/descfmt+
google.golang.org/protobuf/internal/encoding/defval from google.golang.org/protobuf/internal/encoding/tag+
google.golang.org/protobuf/internal/encoding/messageset from google.golang.org/protobuf/encoding/prototext+
google.golang.org/protobuf/internal/encoding/tag from google.golang.org/protobuf/internal/impl
google.golang.org/protobuf/internal/encoding/text from google.golang.org/protobuf/encoding/prototext+
google.golang.org/protobuf/internal/errors from google.golang.org/protobuf/encoding/prototext+
google.golang.org/protobuf/internal/filedesc from google.golang.org/protobuf/internal/encoding/tag+
google.golang.org/protobuf/internal/filetype from google.golang.org/protobuf/runtime/protoimpl
google.golang.org/protobuf/internal/flags from google.golang.org/protobuf/encoding/prototext+
google.golang.org/protobuf/internal/genid from google.golang.org/protobuf/encoding/prototext+
💣 google.golang.org/protobuf/internal/impl from google.golang.org/protobuf/internal/filetype+
google.golang.org/protobuf/internal/order from google.golang.org/protobuf/encoding/prototext+
google.golang.org/protobuf/internal/pragma from google.golang.org/protobuf/encoding/prototext+
google.golang.org/protobuf/internal/set from google.golang.org/protobuf/encoding/prototext
💣 google.golang.org/protobuf/internal/strs from google.golang.org/protobuf/encoding/prototext+
google.golang.org/protobuf/internal/version from google.golang.org/protobuf/runtime/protoimpl
google.golang.org/protobuf/proto from github.com/golang/protobuf/proto+
google.golang.org/protobuf/reflect/protodesc from github.com/golang/protobuf/proto
💣 google.golang.org/protobuf/reflect/protoreflect from github.com/golang/protobuf/proto+
google.golang.org/protobuf/reflect/protoregistry from github.com/golang/protobuf/proto+
google.golang.org/protobuf/runtime/protoiface from github.com/golang/protobuf/proto+
google.golang.org/protobuf/runtime/protoimpl from github.com/golang/protobuf/proto+
google.golang.org/protobuf/types/descriptorpb from google.golang.org/protobuf/reflect/protodesc
google.golang.org/protobuf/types/known/timestamppb from github.com/golang/protobuf/ptypes/timestamp+
gvisor.dev/gvisor/pkg/atomicbitops from gvisor.dev/gvisor/pkg/tcpip+
gvisor.dev/gvisor/pkg/bits from gvisor.dev/gvisor/pkg/bufferv2
💣 gvisor.dev/gvisor/pkg/bufferv2 from gvisor.dev/gvisor/pkg/tcpip+
@ -314,7 +272,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
tailscale.com/tstime from tailscale.com/wgengine/magicsock
💣 tailscale.com/tstime/mono from tailscale.com/net/tstun+
tailscale.com/tstime/rate from tailscale.com/wgengine/filter+
tailscale.com/tsweb from tailscale.com/cmd/tailscaled
tailscale.com/tsweb/varz from tailscale.com/cmd/tailscaled
tailscale.com/types/dnstype from tailscale.com/ipn/ipnlocal+
tailscale.com/types/empty from tailscale.com/control/controlclient+
tailscale.com/types/flagtype from tailscale.com/cmd/tailscaled
@ -357,7 +315,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
tailscale.com/util/sysresources from tailscale.com/wgengine/magicsock
tailscale.com/util/systemd from tailscale.com/control/controlclient+
tailscale.com/util/uniq from tailscale.com/wgengine/magicsock+
tailscale.com/util/vizerror from tailscale.com/tsweb
💣 tailscale.com/util/winutil from tailscale.com/control/controlclient+
W tailscale.com/util/winutil/policy from tailscale.com/ipn/ipnlocal
tailscale.com/version from tailscale.com/derp+
@ -465,7 +422,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
expvar from tailscale.com/derp+
flag from net/http/httptest+
fmt from compress/flate+
go/token from google.golang.org/protobuf/internal/strs
hash from crypto+
hash/adler32 from tailscale.com/ipn/ipnlocal
hash/crc32 from compress/gzip+
@ -504,7 +460,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
regexp from github.com/coreos/go-iptables/iptables+
regexp/syntax from regexp
runtime/debug from github.com/klauspost/compress/zstd+
runtime/metrics from github.com/prometheus/client_golang/prometheus+
runtime/pprof from tailscale.com/log/logheap+
runtime/trace from net/http/pprof
sort from compress/flate+

View File

@ -49,7 +49,7 @@ import (
"tailscale.com/safesocket"
"tailscale.com/smallzstd"
"tailscale.com/syncs"
"tailscale.com/tsweb"
"tailscale.com/tsweb/varz"
"tailscale.com/types/flagtype"
"tailscale.com/types/logger"
"tailscale.com/types/logid"
@ -670,7 +670,7 @@ func newDebugMux() *http.ServeMux {
func servePrometheusMetrics(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
tsweb.VarzHandler(w, r)
varz.Handler(w, r)
clientmetric.WritePrometheusExpositionFormat(w)
}

View File

@ -36,7 +36,7 @@ import (
_ "tailscale.com/ssh/tailssh"
_ "tailscale.com/syncs"
_ "tailscale.com/tailcfg"
_ "tailscale.com/tsweb"
_ "tailscale.com/tsweb/varz"
_ "tailscale.com/types/flagtype"
_ "tailscale.com/types/key"
_ "tailscale.com/types/logger"

View File

@ -36,7 +36,7 @@ import (
_ "tailscale.com/ssh/tailssh"
_ "tailscale.com/syncs"
_ "tailscale.com/tailcfg"
_ "tailscale.com/tsweb"
_ "tailscale.com/tsweb/varz"
_ "tailscale.com/types/flagtype"
_ "tailscale.com/types/key"
_ "tailscale.com/types/logger"

View File

@ -36,7 +36,7 @@ import (
_ "tailscale.com/ssh/tailssh"
_ "tailscale.com/syncs"
_ "tailscale.com/tailcfg"
_ "tailscale.com/tsweb"
_ "tailscale.com/tsweb/varz"
_ "tailscale.com/types/flagtype"
_ "tailscale.com/types/key"
_ "tailscale.com/types/logger"

View File

@ -36,7 +36,7 @@ import (
_ "tailscale.com/ssh/tailssh"
_ "tailscale.com/syncs"
_ "tailscale.com/tailcfg"
_ "tailscale.com/tsweb"
_ "tailscale.com/tsweb/varz"
_ "tailscale.com/types/flagtype"
_ "tailscale.com/types/key"
_ "tailscale.com/types/logger"

View File

@ -43,7 +43,7 @@ import (
_ "tailscale.com/smallzstd"
_ "tailscale.com/syncs"
_ "tailscale.com/tailcfg"
_ "tailscale.com/tsweb"
_ "tailscale.com/tsweb/varz"
_ "tailscale.com/types/flagtype"
_ "tailscale.com/types/key"
_ "tailscale.com/types/logger"

View File

@ -14,6 +14,8 @@ import (
"os"
"runtime"
"tailscale.com/tsweb/promvarz"
"tailscale.com/tsweb/varz"
"tailscale.com/version"
)
@ -51,10 +53,10 @@ func Debugger(mux *http.ServeMux) *DebugHandler {
// index page. The /pprof/ index already covers it.
mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
ret.KVFunc("Uptime", func() any { return Uptime() })
ret.KVFunc("Uptime", func() any { return varz.Uptime() })
ret.KV("Version", version.Long())
ret.Handle("vars", "Metrics (Go)", expvar.Handler())
ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(CombinedVarzHandler))
ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(promvarz.Handler))
ret.Handle("pprof/", "pprof", http.HandlerFunc(pprof.Index))
ret.URL("/debug/pprof/goroutine?debug=1", "Goroutines (collapsed)")
ret.URL("/debug/pprof/goroutine?debug=2", "Goroutines (full)")

View File

@ -0,0 +1,48 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
// Package promvarz combines Prometheus metrics exported by our expvar converter
// (tsweb/varz) with metrics exported by the official Prometheus client.
package promvarz
import (
"fmt"
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/expfmt"
"tailscale.com/tsweb/varz"
)
// Handler returns Prometheus metrics exported by our expvar converter
// and the official Prometheus client.
func Handler(w http.ResponseWriter, r *http.Request) {
if err := gatherNativePrometheusMetrics(w); err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
varz.Handler(w, r)
}
// gatherNativePrometheusMetrics writes metrics from the default
// metric registry in text format.
func gatherNativePrometheusMetrics(w http.ResponseWriter) error {
enc := expfmt.NewEncoder(w, expfmt.FmtText)
mfs, err := prometheus.DefaultGatherer.Gather()
if err != nil {
return fmt.Errorf("could not gather metrics from DefaultGatherer: %w", err)
}
for _, mf := range mfs {
if err := enc.Encode(mf); err != nil {
return fmt.Errorf("could not encode metric %v: %w", mf, err)
}
}
if closer, ok := enc.(expfmt.Closer); ok {
if err := closer.Close(); err != nil {
return err
}
}
return nil
}

View File

@ -0,0 +1,35 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package promvarz
import (
"expvar"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/testutil"
)
func TestHandler(t *testing.T) {
test1 := expvar.NewInt("gauge_promvarz_test_expvar")
test1.Set(42)
test2 := promauto.NewGauge(prometheus.GaugeOpts{Name: "promvarz_test_native"})
test2.Set(4242)
svr := httptest.NewServer(http.HandlerFunc(Handler))
defer svr.Close()
want := `
# TYPE promvarz_test_expvar gauge
promvarz_test_expvar 42
# TYPE promvarz_test_native gauge
promvarz_test_native 4242
`
if err := testutil.ScrapeAndCompare(svr.URL, strings.NewReader(want), "promvarz_test_expvar", "promvarz_test_native"); err != nil {
t.Error(err)
}
}

View File

@ -11,49 +11,25 @@ import (
"errors"
"expvar"
"fmt"
"io"
"net"
"net/http"
_ "net/http/pprof"
"net/netip"
"os"
"path/filepath"
"reflect"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/expfmt"
"go4.org/mem"
"tailscale.com/envknob"
"tailscale.com/metrics"
"tailscale.com/net/tsaddr"
"tailscale.com/tsweb/varz"
"tailscale.com/types/logger"
"tailscale.com/util/vizerror"
"tailscale.com/version"
)
func init() {
expvar.Publish("process_start_unix_time", expvar.Func(func() any { return timeStart.Unix() }))
expvar.Publish("version", expvar.Func(func() any { return version.Long() }))
expvar.Publish("go_version", expvar.Func(func() any { return runtime.Version() }))
expvar.Publish("counter_uptime_sec", expvar.Func(func() any { return int64(Uptime().Seconds()) }))
expvar.Publish("gauge_goroutines", expvar.Func(func() any { return runtime.NumGoroutine() }))
}
const (
gaugePrefix = "gauge_"
counterPrefix = "counter_"
labelMapPrefix = "labelmap_"
)
// prefixesToTrim contains key prefixes to remove when exporting and sorting metrics.
var prefixesToTrim = []string{gaugePrefix, counterPrefix, labelMapPrefix}
// DevMode controls whether extra output in shown, for when the binary is being run in dev mode.
var DevMode bool
@ -143,10 +119,6 @@ func Protected(h http.Handler) http.Handler {
})
}
var timeStart = time.Now()
func Uptime() time.Duration { return time.Since(timeStart).Round(time.Second) }
// Port80Handler is the handler to be given to
// autocert.Manager.HTTPHandler. The inner handler is the mux
// returned by NewMux containing registered /debug handlers.
@ -448,364 +420,8 @@ func Error(code int, msg string, err error) HTTPError {
return HTTPError{Code: code, Msg: msg, Err: err}
}
// WritePrometheusExpvar writes kv to w in Prometheus metrics format.
//
// See VarzHandler for conventions. This is exported primarily for
// people to test their varz.
func WritePrometheusExpvar(w io.Writer, kv expvar.KeyValue) {
writePromExpVar(w, "", kv)
}
type prometheusMetricDetails struct {
Name string
Type string
Label string
}
var prometheusMetricCache sync.Map // string => *prometheusMetricDetails
func prometheusMetric(prefix string, key string) (string, string, string) {
cachekey := prefix + key
if v, ok := prometheusMetricCache.Load(cachekey); ok {
d := v.(*prometheusMetricDetails)
return d.Name, d.Type, d.Label
}
var typ string
var label string
switch {
case strings.HasPrefix(key, gaugePrefix):
typ = "gauge"
key = strings.TrimPrefix(key, gaugePrefix)
case strings.HasPrefix(key, counterPrefix):
typ = "counter"
key = strings.TrimPrefix(key, counterPrefix)
}
if strings.HasPrefix(key, labelMapPrefix) {
key = strings.TrimPrefix(key, labelMapPrefix)
if a, b, ok := strings.Cut(key, "_"); ok {
label, key = a, b
}
}
d := &prometheusMetricDetails{
Name: strings.ReplaceAll(prefix+key, "-", "_"),
Type: typ,
Label: label,
}
prometheusMetricCache.Store(cachekey, d)
return d.Name, d.Type, d.Label
}
func writePromExpVar(w io.Writer, prefix string, kv expvar.KeyValue) {
key := kv.Key
name, typ, label := prometheusMetric(prefix, key)
switch v := kv.Value.(type) {
case *expvar.Int:
if typ == "" {
typ = "counter"
}
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, v.Value())
return
case *expvar.Float:
if typ == "" {
typ = "gauge"
}
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, v.Value())
return
case *metrics.Set:
v.Do(func(kv expvar.KeyValue) {
writePromExpVar(w, name+"_", kv)
})
return
case PrometheusMetricsReflectRooter:
root := v.PrometheusMetricsReflectRoot()
rv := reflect.ValueOf(root)
if rv.Type().Kind() == reflect.Ptr {
if rv.IsNil() {
return
}
rv = rv.Elem()
}
if rv.Type().Kind() != reflect.Struct {
fmt.Fprintf(w, "# skipping expvar %q; unknown root type\n", name)
return
}
foreachExportedStructField(rv, func(fieldOrJSONName, metricType string, rv reflect.Value) {
mname := name + "_" + fieldOrJSONName
switch rv.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Int())
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Uint())
case reflect.Float32, reflect.Float64:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Float())
case reflect.Struct:
if rv.CanAddr() {
// Slight optimization, not copying big structs if they're addressable:
writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Addr().Interface()}})
} else {
writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Interface()}})
}
}
return
})
return
}
if typ == "" {
var funcRet string
if f, ok := kv.Value.(expvar.Func); ok {
v := f()
if ms, ok := v.(runtime.MemStats); ok && name == "memstats" {
writeMemstats(w, &ms)
return
}
if vs, ok := v.(string); ok && strings.HasSuffix(name, "version") {
fmt.Fprintf(w, "%s{version=%q} 1\n", name, vs)
return
}
switch v := v.(type) {
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, uintptr, float32, float64:
fmt.Fprintf(w, "%s %v\n", name, v)
return
}
funcRet = fmt.Sprintf(" returning %T", v)
}
switch kv.Value.(type) {
default:
fmt.Fprintf(w, "# skipping expvar %q (Go type %T%s) with undeclared Prometheus type\n", name, kv.Value, funcRet)
return
case *metrics.LabelMap, *expvar.Map:
// Permit typeless LabelMap and expvar.Map for
// compatibility with old expvar-registered
// metrics.LabelMap.
}
}
switch v := kv.Value.(type) {
case expvar.Func:
val := v()
switch val.(type) {
case float64, int64, int:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val)
default:
fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val)
}
case *metrics.LabelMap:
if typ != "" {
fmt.Fprintf(w, "# TYPE %s %s\n", name, typ)
}
// IntMap uses expvar.Map on the inside, which presorts
// keys. The output ordering is deterministic.
v.Do(func(kv expvar.KeyValue) {
fmt.Fprintf(w, "%s{%s=%q} %v\n", name, v.Label, kv.Key, kv.Value)
})
case *expvar.Map:
if label != "" && typ != "" {
fmt.Fprintf(w, "# TYPE %s %s\n", name, typ)
v.Do(func(kv expvar.KeyValue) {
fmt.Fprintf(w, "%s{%s=%q} %v\n", name, label, kv.Key, kv.Value)
})
} else {
v.Do(func(kv expvar.KeyValue) {
fmt.Fprintf(w, "%s_%s %v\n", name, kv.Key, kv.Value)
})
}
}
}
var sortedKVsPool = &sync.Pool{New: func() any { return new(sortedKVs) }}
// sortedKV is a KeyValue with a sort key.
type sortedKV struct {
expvar.KeyValue
sortKey string // KeyValue.Key with type prefix removed
}
type sortedKVs struct {
kvs []sortedKV
}
// VarzHandler is an HTTP handler to write expvar values into the
// prometheus export format:
//
// https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
//
// It makes the following assumptions:
//
// - *expvar.Int are counters (unless marked as a gauge_; see below)
// - a *tailscale/metrics.Set is descended into, joining keys with
// underscores. So use underscores as your metric names.
// - an expvar named starting with "gauge_" or "counter_" is of that
// Prometheus type, and has that prefix stripped.
// - anything else is untyped and thus not exported.
// - expvar.Func can return an int or int64 (for now) and anything else
// is not exported.
//
// This will evolve over time, or perhaps be replaced.
// VarzHandler writes expvar values as Prometheus metrics.
// TODO: migrate all users to varz.Handler or promvarz.Handler and remove this.
func VarzHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
s := sortedKVsPool.Get().(*sortedKVs)
defer sortedKVsPool.Put(s)
s.kvs = s.kvs[:0]
expvarDo(func(kv expvar.KeyValue) {
s.kvs = append(s.kvs, sortedKV{kv, removeTypePrefixes(kv.Key)})
})
sort.Slice(s.kvs, func(i, j int) bool {
return s.kvs[i].sortKey < s.kvs[j].sortKey
})
for _, e := range s.kvs {
writePromExpVar(w, "", e.KeyValue)
}
varz.Handler(w, r)
}
// CombinedVarzHandler is an HTTP handler for Prometheus metrics that
// combines native metrics with the ones converted from expvar.
func CombinedVarzHandler(w http.ResponseWriter, r *http.Request) {
if err := gatherNativePrometheusMetrics(w); err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
VarzHandler(w, r)
}
// gatherNativePrometheusMetrics writes metrics from the default
// metric registry in text format.
func gatherNativePrometheusMetrics(w http.ResponseWriter) error {
enc := expfmt.NewEncoder(w, expfmt.FmtText)
mfs, err := prometheus.DefaultGatherer.Gather()
if err != nil {
return fmt.Errorf("could not gather metrics from DefaultGatherer: %w", err)
}
for _, mf := range mfs {
if err := enc.Encode(mf); err != nil {
return fmt.Errorf("could not encode metric %v: %w", mf, err)
}
}
if closer, ok := enc.(expfmt.Closer); ok {
if err := closer.Close(); err != nil {
return err
}
}
return nil
}
// PrometheusMetricsReflectRooter is an optional interface that expvar.Var implementations
// can implement to indicate that they should be walked recursively with reflect to find
// sets of fields to export.
type PrometheusMetricsReflectRooter interface {
expvar.Var
// PrometheusMetricsReflectRoot returns the struct or struct pointer to walk.
PrometheusMetricsReflectRoot() any
}
var expvarDo = expvar.Do // pulled out for tests
func writeMemstats(w io.Writer, ms *runtime.MemStats) {
out := func(name, typ string, v uint64, help string) {
if help != "" {
fmt.Fprintf(w, "# HELP memstats_%s %s\n", name, help)
}
fmt.Fprintf(w, "# TYPE memstats_%s %s\nmemstats_%s %v\n", name, typ, name, v)
}
g := func(name string, v uint64, help string) { out(name, "gauge", v, help) }
c := func(name string, v uint64, help string) { out(name, "counter", v, help) }
g("heap_alloc", ms.HeapAlloc, "current bytes of allocated heap objects (up/down smoothly)")
c("total_alloc", ms.TotalAlloc, "cumulative bytes allocated for heap objects")
g("sys", ms.Sys, "total bytes of memory obtained from the OS")
c("mallocs", ms.Mallocs, "cumulative count of heap objects allocated")
c("frees", ms.Frees, "cumulative count of heap objects freed")
c("num_gc", uint64(ms.NumGC), "number of completed GC cycles")
}
// sortedStructField is metadata about a struct field used both for sorting once
// (by structTypeSortedFields) and at serving time (by
// foreachExportedStructField).
type sortedStructField struct {
Index int // index of struct field in struct
Name string // struct field name, or "json" name
SortName string // Name with "foo_" type prefixes removed
MetricType string // the "metrictype" struct tag
StructFieldType *reflect.StructField
}
var structSortedFieldsCache sync.Map // reflect.Type => []sortedStructField
// structTypeSortedFields returns the sorted fields of t, caching as needed.
func structTypeSortedFields(t reflect.Type) []sortedStructField {
if v, ok := structSortedFieldsCache.Load(t); ok {
return v.([]sortedStructField)
}
fields := make([]sortedStructField, 0, t.NumField())
for i, n := 0, t.NumField(); i < n; i++ {
sf := t.Field(i)
name := sf.Name
if v := sf.Tag.Get("json"); v != "" {
v, _, _ = strings.Cut(v, ",")
if v == "-" {
// Skip it, regardless of its metrictype.
continue
}
if v != "" {
name = v
}
}
fields = append(fields, sortedStructField{
Index: i,
Name: name,
SortName: removeTypePrefixes(name),
MetricType: sf.Tag.Get("metrictype"),
StructFieldType: &sf,
})
}
sort.Slice(fields, func(i, j int) bool {
return fields[i].SortName < fields[j].SortName
})
structSortedFieldsCache.Store(t, fields)
return fields
}
// removeTypePrefixes returns s with the first "foo_" prefix in prefixesToTrim
// removed.
func removeTypePrefixes(s string) string {
for _, prefix := range prefixesToTrim {
if trimmed, ok := strings.CutPrefix(s, prefix); ok {
return trimmed
}
}
return s
}
// foreachExportedStructField iterates over the fields in sorted order of
// their name, after removing metric prefixes. This is not necessarily the
// order they were declared in the struct
func foreachExportedStructField(rv reflect.Value, f func(fieldOrJSONName, metricType string, rv reflect.Value)) {
t := rv.Type()
for _, ssf := range structTypeSortedFields(t) {
sf := ssf.StructFieldType
if ssf.MetricType != "" || sf.Type.Kind() == reflect.Struct {
f(ssf.Name, ssf.MetricType, rv.Field(ssf.Index))
} else if sf.Type.Kind() == reflect.Ptr && sf.Type.Elem().Kind() == reflect.Struct {
fv := rv.Field(ssf.Index)
if !fv.IsNil() {
f(ssf.Name, ssf.MetricType, fv.Elem())
}
}
}
}
type expVarPromStructRoot struct{ v any }
func (r expVarPromStructRoot) PrometheusMetricsReflectRoot() any { return r.v }
func (r expVarPromStructRoot) String() string { panic("unused") }
var (
_ PrometheusMetricsReflectRooter = expVarPromStructRoot{}
_ expvar.Var = expVarPromStructRoot{}
)

View File

@ -7,21 +7,17 @@ import (
"bufio"
"context"
"errors"
"expvar"
"fmt"
"net"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"tailscale.com/metrics"
"tailscale.com/tstest"
"tailscale.com/util/vizerror"
"tailscale.com/version"
)
type noopHijacker struct {
@ -370,323 +366,6 @@ func TestHTTPError_Unwrap(t *testing.T) {
}
}
func TestVarzHandler(t *testing.T) {
t.Run("globals_log", func(t *testing.T) {
rec := httptest.NewRecorder()
VarzHandler(rec, httptest.NewRequest("GET", "/", nil))
t.Logf("Got: %s", rec.Body.Bytes())
})
half := new(expvar.Float)
half.Set(0.5)
tests := []struct {
name string
k string // key name
v expvar.Var
want string
}{
{
"int",
"foo",
new(expvar.Int),
"# TYPE foo counter\nfoo 0\n",
},
{
"dash_in_metric_name",
"counter_foo-bar",
new(expvar.Int),
"# TYPE foo_bar counter\nfoo_bar 0\n",
},
{
"int_with_type_counter",
"counter_foo",
new(expvar.Int),
"# TYPE foo counter\nfoo 0\n",
},
{
"int_with_type_gauge",
"gauge_foo",
new(expvar.Int),
"# TYPE foo gauge\nfoo 0\n",
},
{
// For a float = 0.0, Prometheus client_golang outputs "0"
"float_zero",
"foo",
new(expvar.Float),
"# TYPE foo gauge\nfoo 0\n",
},
{
"float_point_5",
"foo",
half,
"# TYPE foo gauge\nfoo 0.5\n",
},
{
"float_with_type_counter",
"counter_foo",
half,
"# TYPE foo counter\nfoo 0.5\n",
},
{
"float_with_type_gauge",
"gauge_foo",
half,
"# TYPE foo gauge\nfoo 0.5\n",
},
{
"metrics_set",
"s",
&metrics.Set{
Map: *(func() *expvar.Map {
m := new(expvar.Map)
m.Init()
m.Add("foo", 1)
m.Add("bar", 2)
return m
})(),
},
"# TYPE s_bar counter\ns_bar 2\n# TYPE s_foo counter\ns_foo 1\n",
},
{
"metrics_set_TODO_gauge_type",
"gauge_s", // TODO(bradfitz): arguably a bug; should pass down type
&metrics.Set{
Map: *(func() *expvar.Map {
m := new(expvar.Map)
m.Init()
m.Add("foo", 1)
m.Add("bar", 2)
return m
})(),
},
"# TYPE s_bar counter\ns_bar 2\n# TYPE s_foo counter\ns_foo 1\n",
},
{
"expvar_map_untyped",
"api_status_code",
func() *expvar.Map {
m := new(expvar.Map)
m.Init()
m.Add("2xx", 100)
m.Add("5xx", 2)
return m
}(),
"api_status_code_2xx 100\napi_status_code_5xx 2\n",
},
{
"func_float64",
"counter_x",
expvar.Func(func() any { return float64(1.2) }),
"# TYPE x counter\nx 1.2\n",
},
{
"func_float64_gauge",
"gauge_y",
expvar.Func(func() any { return float64(1.2) }),
"# TYPE y gauge\ny 1.2\n",
},
{
"func_float64_untyped",
"z",
expvar.Func(func() any { return float64(1.2) }),
"z 1.2\n",
},
{
"metrics_label_map",
"counter_m",
&metrics.LabelMap{
Label: "label",
Map: *(func() *expvar.Map {
m := new(expvar.Map)
m.Init()
m.Add("foo", 1)
m.Add("bar", 2)
return m
})(),
},
"# TYPE m counter\nm{label=\"bar\"} 2\nm{label=\"foo\"} 1\n",
},
{
"metrics_label_map_untyped",
"control_save_config",
(func() *metrics.LabelMap {
m := &metrics.LabelMap{Label: "reason"}
m.Add("new", 1)
m.Add("updated", 1)
m.Add("fun", 1)
return m
})(),
"control_save_config{reason=\"fun\"} 1\ncontrol_save_config{reason=\"new\"} 1\ncontrol_save_config{reason=\"updated\"} 1\n",
},
{
"expvar_label_map",
"counter_labelmap_keyname_m",
func() *expvar.Map {
m := new(expvar.Map)
m.Init()
m.Add("foo", 1)
m.Add("bar", 2)
return m
}(),
"# TYPE m counter\nm{keyname=\"bar\"} 2\nm{keyname=\"foo\"} 1\n",
},
{
"struct_reflect",
"foo",
someExpVarWithJSONAndPromTypes(),
strings.TrimSpace(`
# TYPE foo_AUint16 counter
foo_AUint16 65535
# TYPE foo_AnInt8 counter
foo_AnInt8 127
# TYPE foo_curTemp gauge
foo_curTemp 20.6
# TYPE foo_curX gauge
foo_curX 3
# TYPE foo_nestptr_bar counter
foo_nestptr_bar 20
# TYPE foo_nestptr_foo gauge
foo_nestptr_foo 10
# TYPE foo_nestvalue_bar counter
foo_nestvalue_bar 2
# TYPE foo_nestvalue_foo gauge
foo_nestvalue_foo 1
# TYPE foo_totalY counter
foo_totalY 4
`) + "\n",
},
{
"struct_reflect_nil_root",
"foo",
expvarAdapter{(*SomeStats)(nil)},
"",
},
{
"func_returning_int",
"num_goroutines",
expvar.Func(func() any { return 123 }),
"num_goroutines 123\n",
},
{
"string_version_var",
"foo_version",
expvar.Func(func() any { return "1.2.3-foo15" }),
"foo_version{version=\"1.2.3-foo15\"} 1\n",
},
{
"field_ordering",
"foo",
someExpVarWithFieldNamesSorting(),
strings.TrimSpace(`
# TYPE foo_bar_a gauge
foo_bar_a 1
# TYPE foo_bar_b counter
foo_bar_b 1
# TYPE foo_foo_a gauge
foo_foo_a 1
# TYPE foo_foo_b counter
foo_foo_b 1
`) + "\n",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tstest.Replace(t, &expvarDo, func(f func(expvar.KeyValue)) {
f(expvar.KeyValue{Key: tt.k, Value: tt.v})
})
rec := httptest.NewRecorder()
VarzHandler(rec, httptest.NewRequest("GET", "/", nil))
if got := rec.Body.Bytes(); string(got) != tt.want {
t.Errorf("mismatch\n got: %q\n%s\nwant: %q\n%s\n", got, got, tt.want, tt.want)
}
})
}
}
type SomeNested struct {
FooG int64 `json:"foo" metrictype:"gauge"`
BarC int64 `json:"bar" metrictype:"counter"`
Omit int `json:"-" metrictype:"counter"`
}
type SomeStats struct {
Nested SomeNested `json:"nestvalue"`
NestedPtr *SomeNested `json:"nestptr"`
NestedNilPtr *SomeNested `json:"nestnilptr"`
CurX int `json:"curX" metrictype:"gauge"`
NoMetricType int `json:"noMetric" metrictype:""`
TotalY int64 `json:"totalY,omitempty" metrictype:"counter"`
CurTemp float64 `json:"curTemp" metrictype:"gauge"`
AnInt8 int8 `metrictype:"counter"`
AUint16 uint16 `metrictype:"counter"`
}
// someExpVarWithJSONAndPromTypes returns an expvar.Var that
// implements PrometheusMetricsReflectRooter for TestVarzHandler.
func someExpVarWithJSONAndPromTypes() expvar.Var {
st := &SomeStats{
Nested: SomeNested{
FooG: 1,
BarC: 2,
Omit: 3,
},
NestedPtr: &SomeNested{
FooG: 10,
BarC: 20,
},
CurX: 3,
TotalY: 4,
CurTemp: 20.6,
AnInt8: 127,
AUint16: 65535,
}
return expvarAdapter{st}
}
type expvarAdapter struct {
st *SomeStats
}
func (expvarAdapter) String() string { return "{}" } // expvar JSON; unused in test
func (a expvarAdapter) PrometheusMetricsReflectRoot() any {
return a.st
}
// SomeTestOfFieldNamesSorting demonstrates field
// names that are not in sorted in declaration order, to verify
// that we sort based on field name
type SomeTestOfFieldNamesSorting struct {
FooAG int64 `json:"foo_a" metrictype:"gauge"`
BarAG int64 `json:"bar_a" metrictype:"gauge"`
FooBC int64 `json:"foo_b" metrictype:"counter"`
BarBC int64 `json:"bar_b" metrictype:"counter"`
}
// someExpVarWithFieldNamesSorting returns an expvar.Var that
// implements PrometheusMetricsReflectRooter for TestVarzHandler.
func someExpVarWithFieldNamesSorting() expvar.Var {
st := &SomeTestOfFieldNamesSorting{
FooAG: 1,
BarAG: 1,
FooBC: 1,
BarBC: 1,
}
return expvarAdapter2{st}
}
type expvarAdapter2 struct {
st *SomeTestOfFieldNamesSorting
}
func (expvarAdapter2) String() string { return "{}" } // expvar JSON; unused in test
func (a expvarAdapter2) PrometheusMetricsReflectRoot() any {
return a.st
}
func TestAcceptsEncoding(t *testing.T) {
tests := []struct {
in, enc string
@ -756,45 +435,3 @@ func TestPort80Handler(t *testing.T) {
})
}
}
func TestSortedStructAllocs(t *testing.T) {
f := reflect.ValueOf(struct {
Foo int
Bar int
Baz int
}{})
n := testing.AllocsPerRun(1000, func() {
foreachExportedStructField(f, func(fieldOrJSONName, metricType string, rv reflect.Value) {
// Nothing.
})
})
if n != 0 {
t.Errorf("allocs = %v; want 0", n)
}
}
func TestVarzHandlerSorting(t *testing.T) {
tstest.Replace(t, &expvarDo, func(f func(expvar.KeyValue)) {
f(expvar.KeyValue{Key: "counter_zz", Value: new(expvar.Int)})
f(expvar.KeyValue{Key: "gauge_aa", Value: new(expvar.Int)})
})
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/", nil)
VarzHandler(rec, req)
got := rec.Body.Bytes()
const want = "# TYPE aa gauge\naa 0\n# TYPE zz counter\nzz 0\n"
if string(got) != want {
t.Errorf("got %q; want %q", got, want)
}
rec = new(httptest.ResponseRecorder) // without a body
// Lock in the current number of allocs, to prevent it from growing.
if !version.IsRace() {
allocs := int(testing.AllocsPerRun(1000, func() {
VarzHandler(rec, req)
}))
if max := 13; allocs > max {
t.Errorf("allocs = %v; want max %v", allocs, max)
}
}
}

372
tsweb/varz/varz.go Normal file
View File

@ -0,0 +1,372 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
// Package varz contains code to export metrics in Prometheus format.
package varz
import (
"expvar"
"fmt"
"io"
"net/http"
_ "net/http/pprof"
"reflect"
"runtime"
"sort"
"strings"
"sync"
"time"
"tailscale.com/metrics"
"tailscale.com/version"
)
func init() {
expvar.Publish("process_start_unix_time", expvar.Func(func() any { return timeStart.Unix() }))
expvar.Publish("version", expvar.Func(func() any { return version.Long() }))
expvar.Publish("go_version", expvar.Func(func() any { return runtime.Version() }))
expvar.Publish("counter_uptime_sec", expvar.Func(func() any { return int64(Uptime().Seconds()) }))
expvar.Publish("gauge_goroutines", expvar.Func(func() any { return runtime.NumGoroutine() }))
}
const (
gaugePrefix = "gauge_"
counterPrefix = "counter_"
labelMapPrefix = "labelmap_"
)
// prefixesToTrim contains key prefixes to remove when exporting and sorting metrics.
var prefixesToTrim = []string{gaugePrefix, counterPrefix, labelMapPrefix}
var timeStart = time.Now()
func Uptime() time.Duration { return time.Since(timeStart).Round(time.Second) }
// WritePrometheusExpvar writes kv to w in Prometheus metrics format.
//
// See VarzHandler for conventions. This is exported primarily for
// people to test their varz.
func WritePrometheusExpvar(w io.Writer, kv expvar.KeyValue) {
writePromExpVar(w, "", kv)
}
type prometheusMetricDetails struct {
Name string
Type string
Label string
}
var prometheusMetricCache sync.Map // string => *prometheusMetricDetails
func prometheusMetric(prefix string, key string) (string, string, string) {
cachekey := prefix + key
if v, ok := prometheusMetricCache.Load(cachekey); ok {
d := v.(*prometheusMetricDetails)
return d.Name, d.Type, d.Label
}
var typ string
var label string
switch {
case strings.HasPrefix(key, gaugePrefix):
typ = "gauge"
key = strings.TrimPrefix(key, gaugePrefix)
case strings.HasPrefix(key, counterPrefix):
typ = "counter"
key = strings.TrimPrefix(key, counterPrefix)
}
if strings.HasPrefix(key, labelMapPrefix) {
key = strings.TrimPrefix(key, labelMapPrefix)
if a, b, ok := strings.Cut(key, "_"); ok {
label, key = a, b
}
}
d := &prometheusMetricDetails{
Name: strings.ReplaceAll(prefix+key, "-", "_"),
Type: typ,
Label: label,
}
prometheusMetricCache.Store(cachekey, d)
return d.Name, d.Type, d.Label
}
func writePromExpVar(w io.Writer, prefix string, kv expvar.KeyValue) {
key := kv.Key
name, typ, label := prometheusMetric(prefix, key)
switch v := kv.Value.(type) {
case *expvar.Int:
if typ == "" {
typ = "counter"
}
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, v.Value())
return
case *expvar.Float:
if typ == "" {
typ = "gauge"
}
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, v.Value())
return
case *metrics.Set:
v.Do(func(kv expvar.KeyValue) {
writePromExpVar(w, name+"_", kv)
})
return
case PrometheusMetricsReflectRooter:
root := v.PrometheusMetricsReflectRoot()
rv := reflect.ValueOf(root)
if rv.Type().Kind() == reflect.Ptr {
if rv.IsNil() {
return
}
rv = rv.Elem()
}
if rv.Type().Kind() != reflect.Struct {
fmt.Fprintf(w, "# skipping expvar %q; unknown root type\n", name)
return
}
foreachExportedStructField(rv, func(fieldOrJSONName, metricType string, rv reflect.Value) {
mname := name + "_" + fieldOrJSONName
switch rv.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Int())
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Uint())
case reflect.Float32, reflect.Float64:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Float())
case reflect.Struct:
if rv.CanAddr() {
// Slight optimization, not copying big structs if they're addressable:
writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Addr().Interface()}})
} else {
writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Interface()}})
}
}
return
})
return
}
if typ == "" {
var funcRet string
if f, ok := kv.Value.(expvar.Func); ok {
v := f()
if ms, ok := v.(runtime.MemStats); ok && name == "memstats" {
writeMemstats(w, &ms)
return
}
if vs, ok := v.(string); ok && strings.HasSuffix(name, "version") {
fmt.Fprintf(w, "%s{version=%q} 1\n", name, vs)
return
}
switch v := v.(type) {
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, uintptr, float32, float64:
fmt.Fprintf(w, "%s %v\n", name, v)
return
}
funcRet = fmt.Sprintf(" returning %T", v)
}
switch kv.Value.(type) {
default:
fmt.Fprintf(w, "# skipping expvar %q (Go type %T%s) with undeclared Prometheus type\n", name, kv.Value, funcRet)
return
case *metrics.LabelMap, *expvar.Map:
// Permit typeless LabelMap and expvar.Map for
// compatibility with old expvar-registered
// metrics.LabelMap.
}
}
switch v := kv.Value.(type) {
case expvar.Func:
val := v()
switch val.(type) {
case float64, int64, int:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val)
default:
fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val)
}
case *metrics.LabelMap:
if typ != "" {
fmt.Fprintf(w, "# TYPE %s %s\n", name, typ)
}
// IntMap uses expvar.Map on the inside, which presorts
// keys. The output ordering is deterministic.
v.Do(func(kv expvar.KeyValue) {
fmt.Fprintf(w, "%s{%s=%q} %v\n", name, v.Label, kv.Key, kv.Value)
})
case *expvar.Map:
if label != "" && typ != "" {
fmt.Fprintf(w, "# TYPE %s %s\n", name, typ)
v.Do(func(kv expvar.KeyValue) {
fmt.Fprintf(w, "%s{%s=%q} %v\n", name, label, kv.Key, kv.Value)
})
} else {
v.Do(func(kv expvar.KeyValue) {
fmt.Fprintf(w, "%s_%s %v\n", name, kv.Key, kv.Value)
})
}
}
}
var sortedKVsPool = &sync.Pool{New: func() any { return new(sortedKVs) }}
// sortedKV is a KeyValue with a sort key.
type sortedKV struct {
expvar.KeyValue
sortKey string // KeyValue.Key with type prefix removed
}
type sortedKVs struct {
kvs []sortedKV
}
// Handler is an HTTP handler to write expvar values into the
// prometheus export format:
//
// https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
//
// It makes the following assumptions:
//
// - *expvar.Int are counters (unless marked as a gauge_; see below)
// - a *tailscale/metrics.Set is descended into, joining keys with
// underscores. So use underscores as your metric names.
// - an expvar named starting with "gauge_" or "counter_" is of that
// Prometheus type, and has that prefix stripped.
// - anything else is untyped and thus not exported.
// - expvar.Func can return an int or int64 (for now) and anything else
// is not exported.
//
// This will evolve over time, or perhaps be replaced.
func Handler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
s := sortedKVsPool.Get().(*sortedKVs)
defer sortedKVsPool.Put(s)
s.kvs = s.kvs[:0]
expvarDo(func(kv expvar.KeyValue) {
s.kvs = append(s.kvs, sortedKV{kv, removeTypePrefixes(kv.Key)})
})
sort.Slice(s.kvs, func(i, j int) bool {
return s.kvs[i].sortKey < s.kvs[j].sortKey
})
for _, e := range s.kvs {
writePromExpVar(w, "", e.KeyValue)
}
}
// PrometheusMetricsReflectRooter is an optional interface that expvar.Var implementations
// can implement to indicate that they should be walked recursively with reflect to find
// sets of fields to export.
type PrometheusMetricsReflectRooter interface {
expvar.Var
// PrometheusMetricsReflectRoot returns the struct or struct pointer to walk.
PrometheusMetricsReflectRoot() any
}
var expvarDo = expvar.Do // pulled out for tests
func writeMemstats(w io.Writer, ms *runtime.MemStats) {
out := func(name, typ string, v uint64, help string) {
if help != "" {
fmt.Fprintf(w, "# HELP memstats_%s %s\n", name, help)
}
fmt.Fprintf(w, "# TYPE memstats_%s %s\nmemstats_%s %v\n", name, typ, name, v)
}
g := func(name string, v uint64, help string) { out(name, "gauge", v, help) }
c := func(name string, v uint64, help string) { out(name, "counter", v, help) }
g("heap_alloc", ms.HeapAlloc, "current bytes of allocated heap objects (up/down smoothly)")
c("total_alloc", ms.TotalAlloc, "cumulative bytes allocated for heap objects")
g("sys", ms.Sys, "total bytes of memory obtained from the OS")
c("mallocs", ms.Mallocs, "cumulative count of heap objects allocated")
c("frees", ms.Frees, "cumulative count of heap objects freed")
c("num_gc", uint64(ms.NumGC), "number of completed GC cycles")
}
// sortedStructField is metadata about a struct field used both for sorting once
// (by structTypeSortedFields) and at serving time (by
// foreachExportedStructField).
type sortedStructField struct {
Index int // index of struct field in struct
Name string // struct field name, or "json" name
SortName string // Name with "foo_" type prefixes removed
MetricType string // the "metrictype" struct tag
StructFieldType *reflect.StructField
}
var structSortedFieldsCache sync.Map // reflect.Type => []sortedStructField
// structTypeSortedFields returns the sorted fields of t, caching as needed.
func structTypeSortedFields(t reflect.Type) []sortedStructField {
if v, ok := structSortedFieldsCache.Load(t); ok {
return v.([]sortedStructField)
}
fields := make([]sortedStructField, 0, t.NumField())
for i, n := 0, t.NumField(); i < n; i++ {
sf := t.Field(i)
name := sf.Name
if v := sf.Tag.Get("json"); v != "" {
v, _, _ = strings.Cut(v, ",")
if v == "-" {
// Skip it, regardless of its metrictype.
continue
}
if v != "" {
name = v
}
}
fields = append(fields, sortedStructField{
Index: i,
Name: name,
SortName: removeTypePrefixes(name),
MetricType: sf.Tag.Get("metrictype"),
StructFieldType: &sf,
})
}
sort.Slice(fields, func(i, j int) bool {
return fields[i].SortName < fields[j].SortName
})
structSortedFieldsCache.Store(t, fields)
return fields
}
// removeTypePrefixes returns s with the first "foo_" prefix in prefixesToTrim
// removed.
func removeTypePrefixes(s string) string {
for _, prefix := range prefixesToTrim {
if trimmed, ok := strings.CutPrefix(s, prefix); ok {
return trimmed
}
}
return s
}
// foreachExportedStructField iterates over the fields in sorted order of
// their name, after removing metric prefixes. This is not necessarily the
// order they were declared in the struct
func foreachExportedStructField(rv reflect.Value, f func(fieldOrJSONName, metricType string, rv reflect.Value)) {
t := rv.Type()
for _, ssf := range structTypeSortedFields(t) {
sf := ssf.StructFieldType
if ssf.MetricType != "" || sf.Type.Kind() == reflect.Struct {
f(ssf.Name, ssf.MetricType, rv.Field(ssf.Index))
} else if sf.Type.Kind() == reflect.Ptr && sf.Type.Elem().Kind() == reflect.Struct {
fv := rv.Field(ssf.Index)
if !fv.IsNil() {
f(ssf.Name, ssf.MetricType, fv.Elem())
}
}
}
}
type expVarPromStructRoot struct{ v any }
func (r expVarPromStructRoot) PrometheusMetricsReflectRoot() any { return r.v }
func (r expVarPromStructRoot) String() string { panic("unused") }
var (
_ PrometheusMetricsReflectRooter = expVarPromStructRoot{}
_ expvar.Var = expVarPromStructRoot{}
)

375
tsweb/varz/varz_test.go Normal file
View File

@ -0,0 +1,375 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package varz
import (
"expvar"
"net/http/httptest"
"reflect"
"strings"
"testing"
"tailscale.com/metrics"
"tailscale.com/tstest"
"tailscale.com/version"
)
func TestVarzHandler(t *testing.T) {
t.Run("globals_log", func(t *testing.T) {
rec := httptest.NewRecorder()
Handler(rec, httptest.NewRequest("GET", "/", nil))
t.Logf("Got: %s", rec.Body.Bytes())
})
half := new(expvar.Float)
half.Set(0.5)
tests := []struct {
name string
k string // key name
v expvar.Var
want string
}{
{
"int",
"foo",
new(expvar.Int),
"# TYPE foo counter\nfoo 0\n",
},
{
"dash_in_metric_name",
"counter_foo-bar",
new(expvar.Int),
"# TYPE foo_bar counter\nfoo_bar 0\n",
},
{
"int_with_type_counter",
"counter_foo",
new(expvar.Int),
"# TYPE foo counter\nfoo 0\n",
},
{
"int_with_type_gauge",
"gauge_foo",
new(expvar.Int),
"# TYPE foo gauge\nfoo 0\n",
},
{
// For a float = 0.0, Prometheus client_golang outputs "0"
"float_zero",
"foo",
new(expvar.Float),
"# TYPE foo gauge\nfoo 0\n",
},
{
"float_point_5",
"foo",
half,
"# TYPE foo gauge\nfoo 0.5\n",
},
{
"float_with_type_counter",
"counter_foo",
half,
"# TYPE foo counter\nfoo 0.5\n",
},
{
"float_with_type_gauge",
"gauge_foo",
half,
"# TYPE foo gauge\nfoo 0.5\n",
},
{
"metrics_set",
"s",
&metrics.Set{
Map: *(func() *expvar.Map {
m := new(expvar.Map)
m.Init()
m.Add("foo", 1)
m.Add("bar", 2)
return m
})(),
},
"# TYPE s_bar counter\ns_bar 2\n# TYPE s_foo counter\ns_foo 1\n",
},
{
"metrics_set_TODO_gauge_type",
"gauge_s", // TODO(bradfitz): arguably a bug; should pass down type
&metrics.Set{
Map: *(func() *expvar.Map {
m := new(expvar.Map)
m.Init()
m.Add("foo", 1)
m.Add("bar", 2)
return m
})(),
},
"# TYPE s_bar counter\ns_bar 2\n# TYPE s_foo counter\ns_foo 1\n",
},
{
"expvar_map_untyped",
"api_status_code",
func() *expvar.Map {
m := new(expvar.Map)
m.Init()
m.Add("2xx", 100)
m.Add("5xx", 2)
return m
}(),
"api_status_code_2xx 100\napi_status_code_5xx 2\n",
},
{
"func_float64",
"counter_x",
expvar.Func(func() any { return float64(1.2) }),
"# TYPE x counter\nx 1.2\n",
},
{
"func_float64_gauge",
"gauge_y",
expvar.Func(func() any { return float64(1.2) }),
"# TYPE y gauge\ny 1.2\n",
},
{
"func_float64_untyped",
"z",
expvar.Func(func() any { return float64(1.2) }),
"z 1.2\n",
},
{
"metrics_label_map",
"counter_m",
&metrics.LabelMap{
Label: "label",
Map: *(func() *expvar.Map {
m := new(expvar.Map)
m.Init()
m.Add("foo", 1)
m.Add("bar", 2)
return m
})(),
},
"# TYPE m counter\nm{label=\"bar\"} 2\nm{label=\"foo\"} 1\n",
},
{
"metrics_label_map_untyped",
"control_save_config",
(func() *metrics.LabelMap {
m := &metrics.LabelMap{Label: "reason"}
m.Add("new", 1)
m.Add("updated", 1)
m.Add("fun", 1)
return m
})(),
"control_save_config{reason=\"fun\"} 1\ncontrol_save_config{reason=\"new\"} 1\ncontrol_save_config{reason=\"updated\"} 1\n",
},
{
"expvar_label_map",
"counter_labelmap_keyname_m",
func() *expvar.Map {
m := new(expvar.Map)
m.Init()
m.Add("foo", 1)
m.Add("bar", 2)
return m
}(),
"# TYPE m counter\nm{keyname=\"bar\"} 2\nm{keyname=\"foo\"} 1\n",
},
{
"struct_reflect",
"foo",
someExpVarWithJSONAndPromTypes(),
strings.TrimSpace(`
# TYPE foo_AUint16 counter
foo_AUint16 65535
# TYPE foo_AnInt8 counter
foo_AnInt8 127
# TYPE foo_curTemp gauge
foo_curTemp 20.6
# TYPE foo_curX gauge
foo_curX 3
# TYPE foo_nestptr_bar counter
foo_nestptr_bar 20
# TYPE foo_nestptr_foo gauge
foo_nestptr_foo 10
# TYPE foo_nestvalue_bar counter
foo_nestvalue_bar 2
# TYPE foo_nestvalue_foo gauge
foo_nestvalue_foo 1
# TYPE foo_totalY counter
foo_totalY 4
`) + "\n",
},
{
"struct_reflect_nil_root",
"foo",
expvarAdapter{(*SomeStats)(nil)},
"",
},
{
"func_returning_int",
"num_goroutines",
expvar.Func(func() any { return 123 }),
"num_goroutines 123\n",
},
{
"string_version_var",
"foo_version",
expvar.Func(func() any { return "1.2.3-foo15" }),
"foo_version{version=\"1.2.3-foo15\"} 1\n",
},
{
"field_ordering",
"foo",
someExpVarWithFieldNamesSorting(),
strings.TrimSpace(`
# TYPE foo_bar_a gauge
foo_bar_a 1
# TYPE foo_bar_b counter
foo_bar_b 1
# TYPE foo_foo_a gauge
foo_foo_a 1
# TYPE foo_foo_b counter
foo_foo_b 1
`) + "\n",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tstest.Replace(t, &expvarDo, func(f func(expvar.KeyValue)) {
f(expvar.KeyValue{Key: tt.k, Value: tt.v})
})
rec := httptest.NewRecorder()
Handler(rec, httptest.NewRequest("GET", "/", nil))
if got := rec.Body.Bytes(); string(got) != tt.want {
t.Errorf("mismatch\n got: %q\n%s\nwant: %q\n%s\n", got, got, tt.want, tt.want)
}
})
}
}
type SomeNested struct {
FooG int64 `json:"foo" metrictype:"gauge"`
BarC int64 `json:"bar" metrictype:"counter"`
Omit int `json:"-" metrictype:"counter"`
}
type SomeStats struct {
Nested SomeNested `json:"nestvalue"`
NestedPtr *SomeNested `json:"nestptr"`
NestedNilPtr *SomeNested `json:"nestnilptr"`
CurX int `json:"curX" metrictype:"gauge"`
NoMetricType int `json:"noMetric" metrictype:""`
TotalY int64 `json:"totalY,omitempty" metrictype:"counter"`
CurTemp float64 `json:"curTemp" metrictype:"gauge"`
AnInt8 int8 `metrictype:"counter"`
AUint16 uint16 `metrictype:"counter"`
}
// someExpVarWithJSONAndPromTypes returns an expvar.Var that
// implements PrometheusMetricsReflectRooter for TestVarzHandler.
func someExpVarWithJSONAndPromTypes() expvar.Var {
st := &SomeStats{
Nested: SomeNested{
FooG: 1,
BarC: 2,
Omit: 3,
},
NestedPtr: &SomeNested{
FooG: 10,
BarC: 20,
},
CurX: 3,
TotalY: 4,
CurTemp: 20.6,
AnInt8: 127,
AUint16: 65535,
}
return expvarAdapter{st}
}
type expvarAdapter struct {
st *SomeStats
}
func (expvarAdapter) String() string { return "{}" } // expvar JSON; unused in test
func (a expvarAdapter) PrometheusMetricsReflectRoot() any {
return a.st
}
// SomeTestOfFieldNamesSorting demonstrates field
// names that are not in sorted in declaration order, to verify
// that we sort based on field name
type SomeTestOfFieldNamesSorting struct {
FooAG int64 `json:"foo_a" metrictype:"gauge"`
BarAG int64 `json:"bar_a" metrictype:"gauge"`
FooBC int64 `json:"foo_b" metrictype:"counter"`
BarBC int64 `json:"bar_b" metrictype:"counter"`
}
// someExpVarWithFieldNamesSorting returns an expvar.Var that
// implements PrometheusMetricsReflectRooter for TestVarzHandler.
func someExpVarWithFieldNamesSorting() expvar.Var {
st := &SomeTestOfFieldNamesSorting{
FooAG: 1,
BarAG: 1,
FooBC: 1,
BarBC: 1,
}
return expvarAdapter2{st}
}
type expvarAdapter2 struct {
st *SomeTestOfFieldNamesSorting
}
func (expvarAdapter2) String() string { return "{}" } // expvar JSON; unused in test
func (a expvarAdapter2) PrometheusMetricsReflectRoot() any {
return a.st
}
func TestSortedStructAllocs(t *testing.T) {
f := reflect.ValueOf(struct {
Foo int
Bar int
Baz int
}{})
n := testing.AllocsPerRun(1000, func() {
foreachExportedStructField(f, func(fieldOrJSONName, metricType string, rv reflect.Value) {
// Nothing.
})
})
if n != 0 {
t.Errorf("allocs = %v; want 0", n)
}
}
func TestVarzHandlerSorting(t *testing.T) {
tstest.Replace(t, &expvarDo, func(f func(expvar.KeyValue)) {
f(expvar.KeyValue{Key: "counter_zz", Value: new(expvar.Int)})
f(expvar.KeyValue{Key: "gauge_aa", Value: new(expvar.Int)})
})
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/", nil)
Handler(rec, req)
got := rec.Body.Bytes()
const want = "# TYPE aa gauge\naa 0\n# TYPE zz counter\nzz 0\n"
if string(got) != want {
t.Errorf("got %q; want %q", got, want)
}
rec = new(httptest.ResponseRecorder) // without a body
// Lock in the current number of allocs, to prevent it from growing.
if !version.IsRace() {
allocs := int(testing.AllocsPerRun(1000, func() {
Handler(rec, req)
}))
if max := 13; allocs > max {
t.Errorf("allocs = %v; want max %v", allocs, max)
}
}
}