Fix concurrency limiting?

This commit is contained in:
Tillie Kottmann 2020-10-30 19:26:22 +01:00
parent bcd80689e2
commit 5f332115ff
2 changed files with 9 additions and 21 deletions

View File

@ -1,43 +1,31 @@
// a thing that at least in theory limits concurrency, but i honestly have no clue if it works
package stopandgo package stopandgo
import ( import (
"sync" "sync"
"sync/atomic"
) )
type StopAndGo struct { type StopAndGo struct {
wg sync.WaitGroup wg sync.WaitGroup
maxConcurrency int maxConcurrency int
n int32 guard chan struct{}
} }
func NewStopAndGo(max int) *StopAndGo { func NewStopAndGo(max int) *StopAndGo {
return &StopAndGo{ return &StopAndGo{
maxConcurrency: max, maxConcurrency: max,
guard: make(chan struct{}, max),
} }
} }
func (s *StopAndGo) check() {
if int(atomic.LoadInt32(&s.n)) >= s.maxConcurrency {
s.Wait()
}
}
func (s *StopAndGo) AddN(n int32) {
atomic.AddInt32(&s.n, n)
s.wg.Add(int(n))
// check on a new, separate goroutine so we don't lock everything up
go s.check()
}
func (s *StopAndGo) Add() { func (s *StopAndGo) Add() {
s.AddN(1) s.wg.Add(1)
s.guard <- struct{}{}
} }
func (s *StopAndGo) Done () { func (s *StopAndGo) Done() {
atomic.AddInt32(&s.n, -1)
s.wg.Done() s.wg.Done()
<-s.guard
} }
func (s *StopAndGo) Wait() { func (s *StopAndGo) Wait() {

View File

@ -24,10 +24,10 @@ import (
"time" "time"
) )
const maxConcurrency = 40 const maxConcurrency = 250
var c = &fasthttp.Client{ var c = &fasthttp.Client{
MaxConnsPerHost: 5000,//utils.MaxInt(maxConcurrency + 250, fasthttp.DefaultMaxConnsPerHost), MaxConnsPerHost: utils.MaxInt(maxConcurrency + 250, fasthttp.DefaultMaxConnsPerHost),
TLSConfig: &tls.Config{ TLSConfig: &tls.Config{
InsecureSkipVerify: true, InsecureSkipVerify: true,
}, },