mirror of https://github.com/nyancrimew/goop.git
Add support for fetching repositories via refs
This commit is contained in:
Tillie Kottmann
parent
d51f37de09
commit
79dd76993b
|
|
@ -7,6 +7,7 @@ import (
|
|||
"os"
|
||||
)
|
||||
|
||||
var force bool
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "goop",
|
||||
Short: "goop is a very fast tool to grab sources from exposed .git folders",
|
||||
|
|
@ -16,13 +17,17 @@ var rootCmd = &cobra.Command{
|
|||
if len(args) >= 2 {
|
||||
dir = args[1]
|
||||
}
|
||||
if err := goop.Clone(args[0], dir); err != nil {
|
||||
if err := goop.Clone(args[0], dir, force); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.PersistentFlags().BoolVarP(&force, "force", "f",false, "overrides DIR if it already exists")
|
||||
}
|
||||
|
||||
func Execute() {
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -4,6 +4,9 @@ go 1.14
|
|||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.6.0
|
||||
github.com/gammazero/workerpool v1.1.1
|
||||
github.com/go-git/go-billy/v5 v5.0.0
|
||||
github.com/go-git/go-git/v5 v5.2.0
|
||||
github.com/spf13/cobra v1.1.1
|
||||
github.com/valyala/fasthttp v1.16.0
|
||||
)
|
||||
|
|
|
|||
46
go.sum
46
go.sum
|
|
@ -16,15 +16,18 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
|
|||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94=
|
||||
github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
|
||||
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
|
||||
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
|
||||
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
|
|
@ -37,13 +40,31 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
|
|||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
|
||||
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/gammazero/deque v0.0.0-20200721202602-07291166fe33 h1:UG4wNrJX9xSKnm/Gck5yTbxnOhpNleuE4MQRdmcGySo=
|
||||
github.com/gammazero/deque v0.0.0-20200721202602-07291166fe33/go.mod h1:D90+MBHVc9Sk1lJAbEVgws0eYEurY4mv2TDso3Nxh3w=
|
||||
github.com/gammazero/workerpool v1.1.1 h1:MN29GcZtZZAgzTU+Zk54Y+J9XkE54MoXON/NCZvNulo=
|
||||
github.com/gammazero/workerpool v1.1.1/go.mod h1:5BN0IJVRjSFAypo9QTJCaWdijjNz9Jjl6VFS1PRjCeg=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
|
||||
github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
|
||||
github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM=
|
||||
github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
|
||||
github.com/go-git/go-git v1.0.0 h1:YcN9iDGDoXuIw0vHls6rINwV416HYa0EB2X+RBsyYp4=
|
||||
github.com/go-git/go-git v3.2.0+incompatible h1:puJR2KQIDtZm7uQoF5h+BUFHiU2naU4tfwau90nRIoo=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
|
||||
github.com/go-git/go-git/v5 v5.2.0 h1:YPBLG/3UK1we1ohRkncLjaXWLW+HKp5QNM/jTli2JgI=
|
||||
github.com/go-git/go-git/v5 v5.2.0/go.mod h1:kh02eMX+wdqqxgNMEyq8YgwlIOsDOa9homkUq1PoTMs=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
|
|
@ -94,13 +115,20 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
|
|||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
|
||||
github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg=
|
||||
|
|
@ -110,6 +138,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
|
|||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
|
|
@ -127,6 +156,7 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
|
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
|
|
@ -148,6 +178,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
|
|||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
|
|
@ -167,6 +199,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
|||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
|
|
@ -174,6 +207,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
|
|||
github.com/valyala/fasthttp v1.16.0 h1:9zAqOYLl8Tuy3E5R6ckzGDJ1g8+pw15oQp2iL9Jl6gQ=
|
||||
github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
|
||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
|
||||
github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
|
||||
github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
|
|
@ -183,10 +218,13 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
|
|||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
|
||||
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
|
|
@ -221,6 +259,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
|
|||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
|
||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
|
|
@ -238,12 +277,14 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h
|
|||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y=
|
||||
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
@ -294,11 +335,16 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
|
|||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
|
||||
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
|
||||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package utils
|
|||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
)
|
||||
|
||||
func IsFolder(name string) bool {
|
||||
|
|
@ -14,8 +15,14 @@ func IsFolder(name string) bool {
|
|||
}
|
||||
|
||||
func Exists(name string) bool {
|
||||
_, err := os.Stat(name)
|
||||
return os.IsExist(err)
|
||||
i, err := os.Stat(name)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
if os.IsNotExist(err){
|
||||
return false
|
||||
}
|
||||
return i.IsDir()
|
||||
}
|
||||
|
||||
func IsEmpty(dir string) (bool, error) {
|
||||
|
|
@ -30,4 +37,12 @@ func IsEmpty(dir string) (bool, error) {
|
|||
return true, nil
|
||||
}
|
||||
return false, err // Either not empty or error, suits both cases
|
||||
}
|
||||
}
|
||||
|
||||
func CreateParentFolders(file string) error {
|
||||
dir := path.Dir(file)
|
||||
if !Exists(dir) {
|
||||
return os.MkdirAll(dir, os.ModePerm)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,24 +5,100 @@ import (
|
|||
"crypto/tls"
|
||||
"fmt"
|
||||
"github.com/deletescape/goop/internal/utils"
|
||||
"github.com/go-git/go-billy/v5/osfs"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/cache"
|
||||
"github.com/go-git/go-git/v5/plumbing/format/index"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
"github.com/go-git/go-git/v5/storage/filesystem"
|
||||
"github.com/go-git/go-git/v5/storage/filesystem/dotgit"
|
||||
"github.com/valyala/fasthttp"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var refPrefix = []byte{'r', 'e', 'f', ':'}
|
||||
// TODO: implement limiter
|
||||
const maxConcurrency = 500
|
||||
|
||||
var c = &fasthttp.Client{
|
||||
MaxConnsPerHost: maxConcurrency + 250,
|
||||
TLSConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
}
|
||||
|
||||
func Clone(u, dir string) error {
|
||||
var refPrefix = []byte{'r', 'e', 'f', ':'}
|
||||
var (
|
||||
refRegex = regexp.MustCompile(`(?m)(refs(/[a-zA-Z0-9\-\.\_\*]+)+)`)
|
||||
packRegex = regexp.MustCompile(`(?m)pack-([a-f0-9]{40})\.pack`)
|
||||
objRegex = regexp.MustCompile(`(?m)(^|\s)([a-f0-9]{40})($|\s)`)
|
||||
)
|
||||
var (
|
||||
commonFiles = []string{
|
||||
".gitignore",
|
||||
".gitattributes",
|
||||
".gitmodules",
|
||||
".env",
|
||||
".git/COMMIT_EDITMSG",
|
||||
".git/description",
|
||||
".git/hooks/applypatch-msg.sample",
|
||||
".git/hooks/applypatch-msg.sample",
|
||||
".git/hooks/applypatch-msg.sample",
|
||||
".git/hooks/commit-msg.sample",
|
||||
".git/hooks/post-commit.sample",
|
||||
".git/hooks/post-receive.sample",
|
||||
".git/hooks/post-update.sample",
|
||||
".git/hooks/pre-applypatch.sample",
|
||||
".git/hooks/pre-commit.sample",
|
||||
".git/hooks/pre-push.sample",
|
||||
".git/hooks/pre-rebase.sample",
|
||||
".git/hooks/pre-receive.sample",
|
||||
".git/hooks/prepare-commit-msg.sample",
|
||||
".git/hooks/update.sample",
|
||||
".git/index",
|
||||
".git/info/exclude",
|
||||
".git/objects/info/packs",
|
||||
}
|
||||
commonRefs = []string{
|
||||
".git/FETCH_HEAD",
|
||||
".git/HEAD",
|
||||
".git/ORIG_HEAD",
|
||||
".git/config",
|
||||
".git/info/refs",
|
||||
".git/logs/HEAD",
|
||||
".git/logs/refs/heads/master",
|
||||
".git/logs/refs/heads/main",
|
||||
".git/logs/refs/heads/dev",
|
||||
".git/logs/refs/heads/develop",
|
||||
".git/logs/refs/remotes/origin/HEAD",
|
||||
".git/logs/refs/remotes/origin/master",
|
||||
".git/logs/refs/remotes/origin/main",
|
||||
".git/logs/refs/remotes/origin/dev",
|
||||
".git/logs/refs/remotes/origin/develop",
|
||||
".git/logs/refs/stash",
|
||||
".git/packed-refs",
|
||||
".git/refs/heads/master",
|
||||
".git/refs/heads/main",
|
||||
".git/refs/heads/dev",
|
||||
".git/refs/heads/develop",
|
||||
".git/refs/remotes/origin/HEAD",
|
||||
".git/refs/remotes/origin/master",
|
||||
".git/refs/remotes/origin/main",
|
||||
".git/refs/remotes/origin/dev",
|
||||
".git/refs/remotes/origin/develop",
|
||||
".git/refs/stash",
|
||||
".git/refs/wip/wtree/refs/heads/master", //Magit
|
||||
".git/refs/wip/index/refs/heads/master", //Magit
|
||||
}
|
||||
)
|
||||
|
||||
func Clone(u, dir string, force bool) error {
|
||||
baseUrl := strings.TrimSuffix(u, "/")
|
||||
baseUrl = strings.TrimSuffix(baseUrl, "/HEAD")
|
||||
baseUrl = strings.TrimSuffix(baseUrl, "/.git")
|
||||
|
|
@ -31,17 +107,20 @@ func Clone(u, dir string) error {
|
|||
return err
|
||||
}
|
||||
if parsed.Scheme == "" {
|
||||
parsed.Scheme = "https"
|
||||
parsed.Scheme = "http"
|
||||
}
|
||||
baseUrl = parsed.String()
|
||||
parsed, err = url.Parse(baseUrl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
baseDir := dir
|
||||
if baseDir == "" {
|
||||
baseDir = parsed.Host
|
||||
}
|
||||
|
||||
if !utils.Exists(baseDir) {
|
||||
err := os.MkdirAll(baseDir, os.ModePerm)
|
||||
if err != nil {
|
||||
if err := os.MkdirAll(baseDir, os.ModePerm); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
@ -53,48 +132,189 @@ func Clone(u, dir string) error {
|
|||
return err
|
||||
}
|
||||
if !isEmpty {
|
||||
// todo: add option to force override of the folder here
|
||||
return fmt.Errorf("%s is not empty", baseDir)
|
||||
if force {
|
||||
if err := os.RemoveAll(baseDir); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(baseDir, os.ModePerm); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("%s is not empty", baseDir)
|
||||
}
|
||||
}
|
||||
return FetchGit(baseUrl, baseDir)
|
||||
}
|
||||
|
||||
func DownloadRecursively(u, dir string, files []string, wg *sync.WaitGroup) error {
|
||||
func DownloadRecursively(u, dir string, files []string, wg *sync.WaitGroup) {
|
||||
os.MkdirAll(dir, os.ModePerm)
|
||||
for _, f := range files {
|
||||
f := f
|
||||
uri := utils.Url(u, f)
|
||||
code, body, err := c.Get(nil, uri)
|
||||
fmt.Printf("[-] Fetching %s [%d]\n", uri, code)
|
||||
if err != nil {
|
||||
return err
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
if strings.HasSuffix(f, "/") {
|
||||
wg.Add(1)
|
||||
go func(file, d string, b []byte) {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if !utils.IsHtml(b) {
|
||||
if !utils.IsHtml(body) {
|
||||
fmt.Printf("warning: %s doesn't appear to be an index", uri)
|
||||
return
|
||||
}
|
||||
indexedFiles, err := utils.GetIndexedFiles(b)
|
||||
indexedFiles, err := utils.GetIndexedFiles(body)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
/* return */ DownloadRecursively(uri, utils.Url(d, file), indexedFiles, wg)
|
||||
}(f, dir, body)
|
||||
DownloadRecursively(uri, utils.Url(dir, f), indexedFiles, wg)
|
||||
}()
|
||||
} else {
|
||||
wg.Add(1)
|
||||
go func(file, d string, b []byte) {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if utils.IsHtml(b) {
|
||||
if utils.IsHtml(body) {
|
||||
fmt.Printf("warning: %s doesn't appear to be a git file", uri)
|
||||
return
|
||||
}
|
||||
/* return */ ioutil.WriteFile(utils.Url(d, file), body, os.ModePerm)
|
||||
}(f, dir, body)
|
||||
if err := utils.CreateParentFolders(utils.Url(dir, f)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
if err := ioutil.WriteFile(utils.Url(dir, f), body, os.ModePerm); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func FindRefs(baseUrl, baseDir, path string, wg *sync.WaitGroup) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
uri := utils.Url(baseUrl, path)
|
||||
code, body, err := c.Get(nil, uri)
|
||||
fmt.Printf("[-] Fetching %s [%d]\n", uri, code)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
if code == 200 {
|
||||
if utils.IsHtml(body) {
|
||||
fmt.Printf("warning: %s appears to be an html file, skipping\n", uri)
|
||||
return
|
||||
}
|
||||
if err := utils.CreateParentFolders(utils.Url(baseDir, path)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
if err := ioutil.WriteFile(utils.Url(baseDir, path), body, os.ModePerm); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, ref := range refRegex.FindAll(body, -1) {
|
||||
FindRefs(baseUrl, baseDir, string(ref), wg)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func Download(baseUrl, baseDir, file string, wg *sync.WaitGroup) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
uri := utils.Url(baseUrl, file)
|
||||
code, body, err := c.Get(nil, uri)
|
||||
fmt.Printf("[-] Fetching %s [%d]\n", uri, code)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
if code == 200 {
|
||||
if utils.IsHtml(body) {
|
||||
fmt.Printf("warning: %s appears to be an html file, skipping\n", uri)
|
||||
return
|
||||
}
|
||||
if err := utils.CreateParentFolders(utils.Url(baseDir, file)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
if err := ioutil.WriteFile(utils.Url(baseDir, file), body, os.ModePerm); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func GetReferencedHashes(obj object.Object) []string {
|
||||
var hashes []string
|
||||
switch o := obj.(type) {
|
||||
case *object.Commit:
|
||||
hashes = append(hashes, o.TreeHash.String())
|
||||
for _, p := range o.ParentHashes {
|
||||
hashes = append(hashes, p.String())
|
||||
}
|
||||
case *object.Tree:
|
||||
for _, e := range o.Entries {
|
||||
hashes = append(hashes, e.Hash.String())
|
||||
}
|
||||
case *object.Blob:
|
||||
// pass
|
||||
case *object.Tag:
|
||||
hashes = append(hashes, o.Target.String())
|
||||
}
|
||||
return hashes
|
||||
}
|
||||
|
||||
// TODO: more dedupe stuff
|
||||
func FindObjects(obj, baseUrl, baseDir string, storage *filesystem.ObjectStorage, wg *sync.WaitGroup) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
file := fmt.Sprintf(".git/objects/%s/%s", obj[:2], obj[2:])
|
||||
uri := utils.Url(baseUrl, file)
|
||||
code, body, err := c.Get(nil, uri)
|
||||
fmt.Printf("[-] Fetching %s [%d]\n", uri, code)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
if code == 200 {
|
||||
if utils.IsHtml(body) {
|
||||
fmt.Printf("warning: %s appears to be an html file, skipping\n", uri)
|
||||
return
|
||||
}
|
||||
fullPath := utils.Url(baseDir, file)
|
||||
if err := utils.CreateParentFolders(fullPath); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
if err := ioutil.WriteFile(fullPath, body, os.ModePerm); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
encObj, err := storage.EncodedObject(plumbing.AnyObject, plumbing.NewHash(obj))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
decObj, err := object.DecodeObject(storage, encObj)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return
|
||||
}
|
||||
referencedHashes := GetReferencedHashes(decObj)
|
||||
for _, h := range referencedHashes {
|
||||
FindObjects(h, baseUrl, baseDir, storage, wg)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func FetchGit(baseUrl, baseDir string) error {
|
||||
|
|
@ -125,10 +345,8 @@ func FetchGit(baseUrl, baseDir string) error {
|
|||
}
|
||||
if utils.StringsContain(indexedFiles, "HEAD") {
|
||||
fmt.Println("[-] Fetching .git recursively")
|
||||
var wg sync.WaitGroup
|
||||
if err := DownloadRecursively(utils.Url(baseUrl, ".git/"), utils.Url(baseDir, ".git/"), indexedFiles, &wg); err != nil {
|
||||
return err
|
||||
}
|
||||
wg := sync.WaitGroup{}
|
||||
DownloadRecursively(utils.Url(baseUrl, ".git/"), utils.Url(baseDir, ".git/"), indexedFiles, &wg)
|
||||
wg.Wait()
|
||||
fmt.Println("[-] Running git checkout .")
|
||||
cmd := exec.Command("git", "checkout", ".")
|
||||
|
|
@ -136,5 +354,138 @@ func FetchGit(baseUrl, baseDir string) error {
|
|||
return cmd.Run()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
fmt.Println("[-] Fetching common files")
|
||||
wg := sync.WaitGroup{}
|
||||
for _, f := range commonFiles {
|
||||
Download(baseUrl, baseDir, f, &wg)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
fmt.Println("[-] Finding refs")
|
||||
wg = sync.WaitGroup{}
|
||||
for _, ref := range commonRefs {
|
||||
FindRefs(baseUrl, baseDir, ref, &wg)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
fmt.Println("[-] Finding packs")
|
||||
infoPacksPath := utils.Url(baseDir, ".git/objects/info/packs")
|
||||
if utils.Exists(infoPacksPath) {
|
||||
fmt.Println("exists")
|
||||
infoPacks, err := ioutil.ReadFile(infoPacksPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wg = sync.WaitGroup{}
|
||||
for _, sha1 := range packRegex.FindAll(infoPacks, -1) {
|
||||
Download(baseUrl, baseDir, fmt.Sprintf("./git/objects/pack/pack-%s.idx", sha1), &wg)
|
||||
Download(baseUrl, baseDir, fmt.Sprintf("./git/objects/pack/pack-%s.pack", sha1), &wg)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
fmt.Println("[-] Finding objects")
|
||||
objs := make(map[string]bool) // object "set"
|
||||
//var packed_objs [][]byte
|
||||
|
||||
files := []string{
|
||||
utils.Url(baseDir, ".git/packed-refs"),
|
||||
utils.Url(baseDir, ".git/info/refs"),
|
||||
utils.Url(baseDir, ".git/FETCH_HEAD"),
|
||||
utils.Url(baseDir, ".git/ORIG_HEAD"),
|
||||
}
|
||||
|
||||
if err := filepath.Walk(utils.Url(baseDir, ".git/refs"), func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
files = append(files, path)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := filepath.Walk(utils.Url(baseDir, ".git/logs"), func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
files = append(files, path)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
fmt.Println(f)
|
||||
if !utils.Exists(f) {
|
||||
continue
|
||||
}
|
||||
|
||||
content, err := ioutil.ReadFile(f)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %s\n", err)
|
||||
return err
|
||||
}
|
||||
|
||||
for _, obj := range objRegex.FindAll(content, -1) {
|
||||
objs[strings.TrimSpace(string(obj))] = true
|
||||
}
|
||||
}
|
||||
|
||||
indexPath := utils.Url(baseDir, ".git/index")
|
||||
if utils.Exists(indexPath) {
|
||||
f, err := os.Open(indexPath)
|
||||
defer f.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var idx index.Index
|
||||
decoder := index.NewDecoder(f)
|
||||
if err := decoder.Decode(&idx); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range idx.Entries {
|
||||
objs[entry.Hash.String()] = true
|
||||
}
|
||||
}
|
||||
|
||||
storage := filesystem.NewObjectStorage(dotgit.New(osfs.New(utils.Url(baseDir, ".git"))), &cache.ObjectLRU{MaxSize: 256})
|
||||
if err := storage.ForEachObjectHash(func(hash plumbing.Hash) error {
|
||||
objs[hash.String()] = true
|
||||
encObj, err := storage.EncodedObject(plumbing.AnyObject, hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error: %s\n", err)
|
||||
|
||||
}
|
||||
decObj, err := object.DecodeObject(storage, encObj)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error: %s\n", err)
|
||||
}
|
||||
for _, hash := range GetReferencedHashes(decObj) {
|
||||
objs[hash] = true
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO: find more objects to fetch in pack files and remove packed objects from list of objects to be fetched
|
||||
/*for _, pack := range storage.ObjectPacks() {
|
||||
storage.IterEncodedObjects()
|
||||
}*/
|
||||
|
||||
fmt.Println("[-] Fetching objects")
|
||||
wg = sync.WaitGroup{}
|
||||
for obj := range objs {
|
||||
FindObjects(obj, baseUrl, baseDir, storage, &wg)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
fmt.Println("[-] Running git checkout .")
|
||||
cmd := exec.Command("git", "checkout", ".")
|
||||
cmd.Dir = baseDir
|
||||
return cmd.Run()
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue