logtail: help the server be more efficient

Add content length hints to headers.
The server can use these hints to more efficiently select buffers.

Stop attempting to compress tiny requests.
The bandwidth savings are negligible (and sometimes negative!),
and it makes extra work for the server.

Signed-off-by: Josh Bleecher Snyder <josh@tailscale.com>
This commit is contained in:
Josh Bleecher Snyder 2020-11-23 18:35:49 -08:00 committed by Josh Bleecher Snyder
parent cd6099113f
commit 3a7402aa2d
1 changed files with 20 additions and 6 deletions

View File

@ -14,6 +14,7 @@ import (
"io/ioutil"
"net/http"
"os"
"strconv"
"time"
"tailscale.com/logtail/backoff"
@ -259,8 +260,17 @@ func (l *logger) uploading(ctx context.Context) {
for {
body := l.drainPending()
if l.zstdEncoder != nil {
body = l.zstdEncoder.EncodeAll(body, nil)
origlen := -1 // sentinel value: uncompressed
// Don't attempt to compress tiny bodies; not worth the CPU cycles.
if l.zstdEncoder != nil && len(body) > 256 {
zbody := l.zstdEncoder.EncodeAll(body, nil)
// Only send it compressed if the bandwidth savings are sufficient.
// Just the extra headers associated with enabling compression
// are 50 bytes by themselves.
if len(body)-len(zbody) > 64 {
origlen = len(body)
body = zbody
}
}
for len(body) > 0 {
@ -269,7 +279,7 @@ func (l *logger) uploading(ctx context.Context) {
return
default:
}
uploaded, err := l.upload(ctx, body)
uploaded, err := l.upload(ctx, body, origlen)
if err != nil {
fmt.Fprintf(l.stderr, "logtail: upload: %v\n", err)
}
@ -287,7 +297,10 @@ func (l *logger) uploading(ctx context.Context) {
}
}
func (l *logger) upload(ctx context.Context, body []byte) (uploaded bool, err error) {
// upload uploads body to the log server.
// origlen indicates the pre-compression body length.
// origlen of -1 indicates that the body is not compressed.
func (l *logger) upload(ctx context.Context, body []byte, origlen int) (uploaded bool, err error) {
req, err := http.NewRequest("POST", l.url, bytes.NewReader(body))
if err != nil {
// I know of no conditions under which this could fail.
@ -295,8 +308,9 @@ func (l *logger) upload(ctx context.Context, body []byte) (uploaded bool, err er
// TODO record logs to disk
panic("logtail: cannot build http request: " + err.Error())
}
if l.zstdEncoder != nil {
if origlen != -1 {
req.Header.Add("Content-Encoding", "zstd")
req.Header.Add("Orig-Content-Length", strconv.Itoa(origlen))
}
req.Header["User-Agent"] = nil // not worth writing one; save some bytes
@ -306,7 +320,7 @@ func (l *logger) upload(ctx context.Context, body []byte) (uploaded bool, err er
req = req.WithContext(ctx)
compressedNote := "not-compressed"
if l.zstdEncoder != nil {
if origlen != -1 {
compressedNote = "compressed"
}