mirror of https://github.com/nyancrimew/goop.git
Limit concurrency
This commit is contained in:
Tillie Kottmann
parent
79dd76993b
commit
5cb2134157
1
go.mod
1
go.mod
|
|
@ -4,7 +4,6 @@ go 1.14
|
|||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.6.0
|
||||
github.com/gammazero/workerpool v1.1.1
|
||||
github.com/go-git/go-billy/v5 v5.0.0
|
||||
github.com/go-git/go-git/v5 v5.2.0
|
||||
github.com/spf13/cobra v1.1.1
|
||||
|
|
|
|||
22
go.sum
22
go.sum
|
|
@ -16,6 +16,7 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
|
|||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94=
|
||||
github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
|
||||
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
|
|
@ -23,10 +24,12 @@ github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDa
|
|||
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
|
||||
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
|
||||
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
|
|
@ -42,26 +45,24 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
|
|||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
|
||||
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/gammazero/deque v0.0.0-20200721202602-07291166fe33 h1:UG4wNrJX9xSKnm/Gck5yTbxnOhpNleuE4MQRdmcGySo=
|
||||
github.com/gammazero/deque v0.0.0-20200721202602-07291166fe33/go.mod h1:D90+MBHVc9Sk1lJAbEVgws0eYEurY4mv2TDso3Nxh3w=
|
||||
github.com/gammazero/workerpool v1.1.1 h1:MN29GcZtZZAgzTU+Zk54Y+J9XkE54MoXON/NCZvNulo=
|
||||
github.com/gammazero/workerpool v1.1.1/go.mod h1:5BN0IJVRjSFAypo9QTJCaWdijjNz9Jjl6VFS1PRjCeg=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
|
||||
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
|
||||
github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
|
||||
github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM=
|
||||
github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
|
||||
github.com/go-git/go-git v1.0.0 h1:YcN9iDGDoXuIw0vHls6rINwV416HYa0EB2X+RBsyYp4=
|
||||
github.com/go-git/go-git v3.2.0+incompatible h1:puJR2KQIDtZm7uQoF5h+BUFHiU2naU4tfwau90nRIoo=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12 h1:PbKy9zOy4aAKrJ5pibIRpVO2BXnK1Tlcg+caKI7Ox5M=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
|
||||
github.com/go-git/go-git/v5 v5.2.0 h1:YPBLG/3UK1we1ohRkncLjaXWLW+HKp5QNM/jTli2JgI=
|
||||
github.com/go-git/go-git/v5 v5.2.0/go.mod h1:kh02eMX+wdqqxgNMEyq8YgwlIOsDOa9homkUq1PoTMs=
|
||||
|
|
@ -83,6 +84,7 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
|
|||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
|
|
@ -138,6 +140,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
|
|||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
|
|
@ -156,12 +159,15 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
|
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
|
|
@ -199,6 +205,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
|||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
|
|
@ -290,6 +297,7 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSK
|
|||
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
@ -336,6 +344,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
|
|||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
|
|
@ -346,6 +355,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
|
|||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
// a thing that at least in theory limits concurrency, but i honestly have no clue if it works
|
||||
package stopandgo
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
type StopAndGo struct {
|
||||
wg sync.WaitGroup
|
||||
maxConcurrency int
|
||||
n int32
|
||||
}
|
||||
|
||||
func NewStopAndGo(max int) *StopAndGo {
|
||||
return &StopAndGo{
|
||||
maxConcurrency: max,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StopAndGo) check() {
|
||||
if int(atomic.LoadInt32(&s.n)) >= s.maxConcurrency {
|
||||
s.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StopAndGo) AddN(n int32) {
|
||||
atomic.AddInt32(&s.n, n)
|
||||
s.wg.Add(int(n))
|
||||
// check on a new, separate goroutine so we don't lock everything up
|
||||
go s.check()
|
||||
}
|
||||
|
||||
func (s *StopAndGo) Add() {
|
||||
s.AddN(1)
|
||||
}
|
||||
|
||||
func (s *StopAndGo) Done () {
|
||||
atomic.AddInt32(&s.n, -1)
|
||||
s.wg.Done()
|
||||
}
|
||||
|
||||
func (s *StopAndGo) Wait() {
|
||||
s.wg.Wait()
|
||||
s.wg = sync.WaitGroup{}
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package utils
|
||||
|
||||
func MaxInt(x, y int) int {
|
||||
if x > y {
|
||||
return x
|
||||
}
|
||||
return y
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"bytes"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"github.com/deletescape/goop/internal/stopandgo"
|
||||
"github.com/deletescape/goop/internal/utils"
|
||||
"github.com/go-git/go-billy/v5/osfs"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
|
|
@ -20,17 +21,18 @@ import (
|
|||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TODO: implement limiter
|
||||
const maxConcurrency = 500
|
||||
const maxConcurrency = 40
|
||||
|
||||
var c = &fasthttp.Client{
|
||||
MaxConnsPerHost: maxConcurrency + 250,
|
||||
MaxConnsPerHost: utils.MaxInt(maxConcurrency + 250, fasthttp.DefaultMaxConnsPerHost),
|
||||
TLSConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
NoDefaultUserAgentHeader: true,
|
||||
MaxConnWaitTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
var refPrefix = []byte{'r', 'e', 'f', ':'}
|
||||
|
|
@ -98,6 +100,8 @@ var (
|
|||
}
|
||||
)
|
||||
|
||||
var sag = stopandgo.NewStopAndGo(maxConcurrency)
|
||||
|
||||
func Clone(u, dir string, force bool) error {
|
||||
baseUrl := strings.TrimSuffix(u, "/")
|
||||
baseUrl = strings.TrimSuffix(baseUrl, "/HEAD")
|
||||
|
|
@ -146,7 +150,7 @@ func Clone(u, dir string, force bool) error {
|
|||
return FetchGit(baseUrl, baseDir)
|
||||
}
|
||||
|
||||
func DownloadRecursively(u, dir string, files []string, wg *sync.WaitGroup) {
|
||||
func DownloadRecursively(u, dir string, files []string) {
|
||||
os.MkdirAll(dir, os.ModePerm)
|
||||
for _, f := range files {
|
||||
f := f
|
||||
|
|
@ -158,9 +162,9 @@ func DownloadRecursively(u, dir string, files []string, wg *sync.WaitGroup) {
|
|||
return
|
||||
}
|
||||
if strings.HasSuffix(f, "/") {
|
||||
wg.Add(1)
|
||||
sag.Add()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer sag.Done()
|
||||
if !utils.IsHtml(body) {
|
||||
fmt.Printf("warning: %s doesn't appear to be an index", uri)
|
||||
return
|
||||
|
|
@ -170,12 +174,12 @@ func DownloadRecursively(u, dir string, files []string, wg *sync.WaitGroup) {
|
|||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
DownloadRecursively(uri, utils.Url(dir, f), indexedFiles, wg)
|
||||
DownloadRecursively(uri, utils.Url(dir, f), indexedFiles)
|
||||
}()
|
||||
} else {
|
||||
wg.Add(1)
|
||||
sag.Add()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer sag.Done()
|
||||
if utils.IsHtml(body) {
|
||||
fmt.Printf("warning: %s doesn't appear to be a git file", uri)
|
||||
return
|
||||
|
|
@ -192,10 +196,10 @@ func DownloadRecursively(u, dir string, files []string, wg *sync.WaitGroup) {
|
|||
}
|
||||
}
|
||||
|
||||
func FindRefs(baseUrl, baseDir, path string, wg *sync.WaitGroup) {
|
||||
wg.Add(1)
|
||||
func FindRefs(baseUrl, baseDir, path string) {
|
||||
sag.Add()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer sag.Done()
|
||||
uri := utils.Url(baseUrl, path)
|
||||
code, body, err := c.Get(nil, uri)
|
||||
fmt.Printf("[-] Fetching %s [%d]\n", uri, code)
|
||||
|
|
@ -218,16 +222,16 @@ func FindRefs(baseUrl, baseDir, path string, wg *sync.WaitGroup) {
|
|||
}
|
||||
|
||||
for _, ref := range refRegex.FindAll(body, -1) {
|
||||
FindRefs(baseUrl, baseDir, string(ref), wg)
|
||||
FindRefs(baseUrl, baseDir, string(ref))
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func Download(baseUrl, baseDir, file string, wg *sync.WaitGroup) {
|
||||
wg.Add(1)
|
||||
func Download(baseUrl, baseDir, file string) {
|
||||
sag.Add()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer sag.Done()
|
||||
uri := utils.Url(baseUrl, file)
|
||||
code, body, err := c.Get(nil, uri)
|
||||
fmt.Printf("[-] Fetching %s [%d]\n", uri, code)
|
||||
|
|
@ -272,10 +276,10 @@ func GetReferencedHashes(obj object.Object) []string {
|
|||
}
|
||||
|
||||
// TODO: more dedupe stuff
|
||||
func FindObjects(obj, baseUrl, baseDir string, storage *filesystem.ObjectStorage, wg *sync.WaitGroup) {
|
||||
wg.Add(1)
|
||||
func FindObjects(obj, baseUrl, baseDir string, storage *filesystem.ObjectStorage) {
|
||||
sag.Add()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer sag.Done()
|
||||
file := fmt.Sprintf(".git/objects/%s/%s", obj[:2], obj[2:])
|
||||
uri := utils.Url(baseUrl, file)
|
||||
code, body, err := c.Get(nil, uri)
|
||||
|
|
@ -311,7 +315,7 @@ func FindObjects(obj, baseUrl, baseDir string, storage *filesystem.ObjectStorage
|
|||
}
|
||||
referencedHashes := GetReferencedHashes(decObj)
|
||||
for _, h := range referencedHashes {
|
||||
FindObjects(h, baseUrl, baseDir, storage, wg)
|
||||
FindObjects(h, baseUrl, baseDir, storage)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
|
@ -345,9 +349,8 @@ func FetchGit(baseUrl, baseDir string) error {
|
|||
}
|
||||
if utils.StringsContain(indexedFiles, "HEAD") {
|
||||
fmt.Println("[-] Fetching .git recursively")
|
||||
wg := sync.WaitGroup{}
|
||||
DownloadRecursively(utils.Url(baseUrl, ".git/"), utils.Url(baseDir, ".git/"), indexedFiles, &wg)
|
||||
wg.Wait()
|
||||
DownloadRecursively(utils.Url(baseUrl, ".git/"), utils.Url(baseDir, ".git/"), indexedFiles)
|
||||
sag.Wait()
|
||||
fmt.Println("[-] Running git checkout .")
|
||||
cmd := exec.Command("git", "checkout", ".")
|
||||
cmd.Dir = baseDir
|
||||
|
|
@ -356,18 +359,16 @@ func FetchGit(baseUrl, baseDir string) error {
|
|||
}
|
||||
|
||||
fmt.Println("[-] Fetching common files")
|
||||
wg := sync.WaitGroup{}
|
||||
for _, f := range commonFiles {
|
||||
Download(baseUrl, baseDir, f, &wg)
|
||||
Download(baseUrl, baseDir, f)
|
||||
}
|
||||
wg.Wait()
|
||||
sag.Wait()
|
||||
|
||||
fmt.Println("[-] Finding refs")
|
||||
wg = sync.WaitGroup{}
|
||||
for _, ref := range commonRefs {
|
||||
FindRefs(baseUrl, baseDir, ref, &wg)
|
||||
FindRefs(baseUrl, baseDir, ref)
|
||||
}
|
||||
wg.Wait()
|
||||
sag.Wait()
|
||||
|
||||
fmt.Println("[-] Finding packs")
|
||||
infoPacksPath := utils.Url(baseDir, ".git/objects/info/packs")
|
||||
|
|
@ -377,12 +378,11 @@ func FetchGit(baseUrl, baseDir string) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wg = sync.WaitGroup{}
|
||||
for _, sha1 := range packRegex.FindAll(infoPacks, -1) {
|
||||
Download(baseUrl, baseDir, fmt.Sprintf("./git/objects/pack/pack-%s.idx", sha1), &wg)
|
||||
Download(baseUrl, baseDir, fmt.Sprintf("./git/objects/pack/pack-%s.pack", sha1), &wg)
|
||||
Download(baseUrl, baseDir, fmt.Sprintf("./git/objects/pack/pack-%s.idx", sha1))
|
||||
Download(baseUrl, baseDir, fmt.Sprintf("./git/objects/pack/pack-%s.pack", sha1))
|
||||
}
|
||||
wg.Wait()
|
||||
sag.Wait()
|
||||
}
|
||||
|
||||
fmt.Println("[-] Finding objects")
|
||||
|
|
@ -420,7 +420,6 @@ func FetchGit(baseUrl, baseDir string) error {
|
|||
}
|
||||
|
||||
for _, f := range files {
|
||||
fmt.Println(f)
|
||||
if !utils.Exists(f) {
|
||||
continue
|
||||
}
|
||||
|
|
@ -478,11 +477,10 @@ func FetchGit(baseUrl, baseDir string) error {
|
|||
}*/
|
||||
|
||||
fmt.Println("[-] Fetching objects")
|
||||
wg = sync.WaitGroup{}
|
||||
for obj := range objs {
|
||||
FindObjects(obj, baseUrl, baseDir, storage, &wg)
|
||||
FindObjects(obj, baseUrl, baseDir, storage)
|
||||
}
|
||||
wg.Wait()
|
||||
sag.Wait()
|
||||
|
||||
fmt.Println("[-] Running git checkout .")
|
||||
cmd := exec.Command("git", "checkout", ".")
|
||||
|
|
|
|||
Loading…
Reference in New Issue