From 5237f79740421c920f03d9ad489f2a2289af18f1 Mon Sep 17 00:00:00 2001 From: ginuerzh Date: Tue, 20 Sep 2022 17:54:03 +0800 Subject: [PATCH] add chain hop --- admission/admission.go | 17 +++++ auth/auth.go | 18 ++++++ bypass/bypass.go | 16 +++++ chain/hop.go | 125 ++++++++++++++++++++++++++++++++++--- config/config.go | 10 +++ config/parsing/parse.go | 44 +++++++++++++ go.mod | 2 +- go.sum | 4 +- hosts/hosts.go | 16 +++++ internal/loader/http.go | 71 +++++++++++++++++++++ internal/loader/redis.go | 4 +- limiter/conn/conn.go | 16 +++++ limiter/rate/rate.go | 16 +++++ limiter/traffic/traffic.go | 16 +++++ 14 files changed, 364 insertions(+), 11 deletions(-) create mode 100644 internal/loader/http.go diff --git a/admission/admission.go b/admission/admission.go index eca6643..8d7e278 100644 --- a/admission/admission.go +++ b/admission/admission.go @@ -20,6 +20,7 @@ type options struct { matchers []string fileLoader loader.Loader redisLoader loader.Loader + httpLoader loader.Loader period time.Duration logger logger.Logger } @@ -56,6 +57,12 @@ func RedisLoaderOption(redisLoader loader.Loader) Option { } } +func HTTPLoaderOption(httpLoader loader.Loader) Option { + return func(opts *options) { + opts.httpLoader = httpLoader + } +} + func LoggerOption(logger logger.Logger) Option { return func(opts *options) { opts.logger = logger @@ -200,6 +207,16 @@ func (p *admission) load(ctx context.Context) (patterns []string, err error) { } } + if p.options.httpLoader != nil { + r, er := p.options.httpLoader.Load(ctx) + if er != nil { + p.options.logger.Warnf("http loader: %v", er) + } + if v, _ := p.parsePatterns(r); v != nil { + patterns = append(patterns, v...) + } + } + p.options.logger.Debugf("load items %d", len(patterns)) return } diff --git a/auth/auth.go b/auth/auth.go index c3f2e00..0f9b16b 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -17,6 +17,7 @@ type options struct { auths map[string]string fileLoader loader.Loader redisLoader loader.Loader + httpLoader loader.Loader period time.Duration logger logger.Logger } @@ -47,6 +48,12 @@ func RedisLoaderOption(redisLoader loader.Loader) Option { } } +func HTTPLoaderOption(httpLoader loader.Loader) Option { + return func(opts *options) { + opts.httpLoader = httpLoader + } +} + func LoggerOption(logger logger.Logger) Option { return func(opts *options) { opts.logger = logger @@ -179,6 +186,17 @@ func (p *authenticator) load(ctx context.Context) (m map[string]string, err erro } } } + if p.options.httpLoader != nil { + r, er := p.options.httpLoader.Load(ctx) + if er != nil { + p.options.logger.Warnf("http loader: %v", er) + } + if auths, _ := p.parseAuths(r); auths != nil { + for k, v := range auths { + m[k] = v + } + } + } p.options.logger.Debugf("load items %d", len(m)) return diff --git a/bypass/bypass.go b/bypass/bypass.go index c1cd701..6c7fd59 100644 --- a/bypass/bypass.go +++ b/bypass/bypass.go @@ -20,6 +20,7 @@ type options struct { matchers []string fileLoader loader.Loader redisLoader loader.Loader + httpLoader loader.Loader period time.Duration logger logger.Logger } @@ -56,6 +57,12 @@ func RedisLoaderOption(redisLoader loader.Loader) Option { } } +func HTTPLoaderOption(httpLoader loader.Loader) Option { + return func(opts *options) { + opts.httpLoader = httpLoader + } +} + func LoggerOption(logger logger.Logger) Option { return func(opts *options) { opts.logger = logger @@ -195,6 +202,15 @@ func (bp *bypass) load(ctx context.Context) (patterns []string, err error) { } } } + if bp.options.httpLoader != nil { + r, er := bp.options.httpLoader.Load(ctx) + if er != nil { + bp.options.logger.Warnf("http loader: %v", er) + } + if v, _ := bp.parsePatterns(r); v != nil { + patterns = append(patterns, v...) + } + } bp.options.logger.Debugf("load items %d", len(patterns)) return diff --git a/chain/hop.go b/chain/hop.go index 72816c0..a57e0e4 100644 --- a/chain/hop.go +++ b/chain/hop.go @@ -2,15 +2,25 @@ package chain import ( "context" + "io" + "sync" + "time" "github.com/go-gost/core/bypass" "github.com/go-gost/core/chain" + "github.com/go-gost/core/logger" "github.com/go-gost/core/selector" + "github.com/go-gost/x/internal/loader" ) type HopOptions struct { - bypass bypass.Bypass - selector selector.Selector[*chain.Node] + bypass bypass.Bypass + selector selector.Selector[*chain.Node] + fileLoader loader.Loader + httpLoader loader.Loader + redisLoader loader.Loader + period time.Duration + logger logger.Logger } type HopOption func(*HopOptions) @@ -27,9 +37,41 @@ func SelectorHopOption(s selector.Selector[*chain.Node]) HopOption { } } +func FileLoaderHopOption(fileLoader loader.Loader) HopOption { + return func(opts *HopOptions) { + opts.fileLoader = fileLoader + } +} + +func RedisLoaderHopOption(redisLoader loader.Loader) HopOption { + return func(opts *HopOptions) { + opts.redisLoader = redisLoader + } +} + +func HTTPLoaderHopOption(httpLoader loader.Loader) HopOption { + return func(opts *HopOptions) { + opts.httpLoader = httpLoader + } +} + +func ReloadPeriodHopOption(period time.Duration) HopOption { + return func(opts *HopOptions) { + opts.period = period + } +} + +func LoggerHopOption(logger logger.Logger) HopOption { + return func(opts *HopOptions) { + opts.logger = logger + } +} + type chainHop struct { - nodes []*chain.Node - options HopOptions + nodes []*chain.Node + options HopOptions + cancelFunc context.CancelFunc + mu sync.RWMutex } func NewChainHop(nodes []*chain.Node, opts ...HopOption) chain.Hop { @@ -40,10 +82,21 @@ func NewChainHop(nodes []*chain.Node, opts ...HopOption) chain.Hop { } } - return &chainHop{ - nodes: nodes, - options: options, + ctx, cancel := context.WithCancel(context.TODO()) + + hop := &chainHop{ + nodes: nodes, + options: options, + cancelFunc: cancel, } + if err := hop.reload(ctx); err != nil { + options.logger.Warnf("reload: %v", err) + } + if hop.options.period > 0 { + go hop.periodReload(ctx) + } + + return hop } func (p *chainHop) Nodes() []*chain.Node { @@ -85,3 +138,61 @@ func (p *chainHop) Select(ctx context.Context, opts ...chain.SelectOption) *chai } return nodes[0] } + +func (p *chainHop) periodReload(ctx context.Context) error { + period := p.options.period + if period < time.Second { + period = time.Second + } + ticker := time.NewTicker(period) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := p.reload(ctx); err != nil { + p.options.logger.Warnf("reload: %v", err) + // return err + } + case <-ctx.Done(): + return ctx.Err() + } + } +} + +func (p *chainHop) reload(ctx context.Context) error { + _, err := p.load(ctx) + if err != nil { + return err + } + + return nil +} + +func (p *chainHop) load(ctx context.Context) (data []byte, err error) { + if p.options.fileLoader != nil { + r, er := p.options.fileLoader.Load(ctx) + if er != nil { + p.options.logger.Warnf("file loader: %v", er) + } + return io.ReadAll(r) + } + + if p.options.redisLoader != nil { + r, er := p.options.redisLoader.Load(ctx) + if er != nil { + p.options.logger.Warnf("redis loader: %v", er) + } + return io.ReadAll(r) + } + + if p.options.httpLoader != nil { + r, er := p.options.redisLoader.Load(ctx) + if er != nil { + p.options.logger.Warnf("http loader: %v", er) + } + return io.ReadAll(r) + } + + return +} diff --git a/config/config.go b/config/config.go index a63c353..4f8eb07 100644 --- a/config/config.go +++ b/config/config.go @@ -84,6 +84,7 @@ type AutherConfig struct { Reload time.Duration `yaml:",omitempty" json:"reload,omitempty"` File *FileLoader `yaml:",omitempty" json:"file,omitempty"` Redis *RedisLoader `yaml:",omitempty" json:"redis,omitempty"` + HTTP *HTTPLoader `yaml:"http,omitempty" json:"http,omitempty"` } type AuthConfig struct { @@ -106,6 +107,7 @@ type AdmissionConfig struct { Reload time.Duration `yaml:",omitempty" json:"reload,omitempty"` File *FileLoader `yaml:",omitempty" json:"file,omitempty"` Redis *RedisLoader `yaml:",omitempty" json:"redis,omitempty"` + HTTP *HTTPLoader `yaml:"http,omitempty" json:"http,omitempty"` } type BypassConfig struct { @@ -117,6 +119,7 @@ type BypassConfig struct { Reload time.Duration `yaml:",omitempty" json:"reload,omitempty"` File *FileLoader `yaml:",omitempty" json:"file,omitempty"` Redis *RedisLoader `yaml:",omitempty" json:"redis,omitempty"` + HTTP *HTTPLoader `yaml:"http,omitempty" json:"http,omitempty"` } type FileLoader struct { @@ -131,6 +134,11 @@ type RedisLoader struct { Type string `yaml:",omitempty" json:"type,omitempty"` } +type HTTPLoader struct { + URL string `yaml:"url" json:"url"` + Timeout time.Duration `yaml:",omitempty" json:"timeout,omitempty"` +} + type NameserverConfig struct { Addr string `json:"addr"` Chain string `yaml:",omitempty" json:"chain,omitempty"` @@ -158,6 +166,7 @@ type HostsConfig struct { Reload time.Duration `yaml:",omitempty" json:"reload,omitempty"` File *FileLoader `yaml:",omitempty" json:"file,omitempty"` Redis *RedisLoader `yaml:",omitempty" json:"redis,omitempty"` + HTTP *HTTPLoader `yaml:"http,omitempty" json:"http,omitempty"` } type RecorderConfig struct { @@ -190,6 +199,7 @@ type LimiterConfig struct { Reload time.Duration `yaml:",omitempty" json:"reload,omitempty"` File *FileLoader `yaml:",omitempty" json:"file,omitempty"` Redis *RedisLoader `yaml:",omitempty" json:"redis,omitempty"` + HTTP *HTTPLoader `yaml:"http,omitempty" json:"http,omitempty"` } type ListenerConfig struct { diff --git a/config/parsing/parse.go b/config/parsing/parse.go index 9026256..6340e3a 100644 --- a/config/parsing/parse.go +++ b/config/parsing/parse.go @@ -70,6 +70,12 @@ func ParseAuther(cfg *config.AutherConfig) auth.Authenticator { loader.KeyRedisLoaderOption(cfg.Redis.Key), ))) } + if cfg.HTTP != nil && cfg.HTTP.URL != "" { + opts = append(opts, auth_impl.HTTPLoaderOption(loader.HTTPLoader( + cfg.HTTP.URL, + loader.TimeoutHTTPLoaderOption(cfg.HTTP.Timeout), + ))) + } return auth_impl.NewAuthenticator(opts...) } @@ -172,6 +178,13 @@ func ParseAdmission(cfg *config.AdmissionConfig) admission.Admission { loader.KeyRedisLoaderOption(cfg.Redis.Key), ))) } + if cfg.HTTP != nil && cfg.HTTP.URL != "" { + opts = append(opts, admission_impl.HTTPLoaderOption(loader.HTTPLoader( + cfg.HTTP.URL, + loader.TimeoutHTTPLoaderOption(cfg.HTTP.Timeout), + ))) + } + return admission_impl.NewAdmission(opts...) } @@ -200,6 +213,13 @@ func ParseBypass(cfg *config.BypassConfig) bypass.Bypass { loader.KeyRedisLoaderOption(cfg.Redis.Key), ))) } + if cfg.HTTP != nil && cfg.HTTP.URL != "" { + opts = append(opts, bypass_impl.HTTPLoaderOption(loader.HTTPLoader( + cfg.HTTP.URL, + loader.TimeoutHTTPLoaderOption(cfg.HTTP.Timeout), + ))) + } + return bypass_impl.NewBypass(opts...) } @@ -280,6 +300,12 @@ func ParseHosts(cfg *config.HostsConfig) hosts.HostMapper { ))) } } + if cfg.HTTP != nil && cfg.HTTP.URL != "" { + opts = append(opts, xhosts.HTTPLoaderOption(loader.HTTPLoader( + cfg.HTTP.URL, + loader.TimeoutHTTPLoaderOption(cfg.HTTP.Timeout), + ))) + } return xhosts.NewHostMapper(opts...) } @@ -359,6 +385,12 @@ func ParseTrafficLimiter(cfg *config.LimiterConfig) (lim traffic.TrafficLimiter) ))) } } + if cfg.HTTP != nil && cfg.HTTP.URL != "" { + opts = append(opts, xtraffic.HTTPLoaderOption(loader.HTTPLoader( + cfg.HTTP.URL, + loader.TimeoutHTTPLoaderOption(cfg.HTTP.Timeout), + ))) + } opts = append(opts, xtraffic.LimitsOption(cfg.Limits...), xtraffic.ReloadPeriodOption(cfg.Reload), @@ -399,6 +431,12 @@ func ParseConnLimiter(cfg *config.LimiterConfig) (lim conn.ConnLimiter) { ))) } } + if cfg.HTTP != nil && cfg.HTTP.URL != "" { + opts = append(opts, xconn.HTTPLoaderOption(loader.HTTPLoader( + cfg.HTTP.URL, + loader.TimeoutHTTPLoaderOption(cfg.HTTP.Timeout), + ))) + } opts = append(opts, xconn.LimitsOption(cfg.Limits...), xconn.ReloadPeriodOption(cfg.Reload), @@ -439,6 +477,12 @@ func ParseRateLimiter(cfg *config.LimiterConfig) (lim rate.RateLimiter) { ))) } } + if cfg.HTTP != nil && cfg.HTTP.URL != "" { + opts = append(opts, xrate.HTTPLoaderOption(loader.HTTPLoader( + cfg.HTTP.URL, + loader.TimeoutHTTPLoaderOption(cfg.HTTP.Timeout), + ))) + } opts = append(opts, xrate.LimitsOption(cfg.Limits...), xrate.ReloadPeriodOption(cfg.Reload), diff --git a/go.mod b/go.mod index 81e06dd..40a0e82 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d github.com/gin-contrib/cors v1.3.1 github.com/gin-gonic/gin v1.7.7 - github.com/go-gost/core v0.0.0-20220914115321-50d443049f3b + github.com/go-gost/core v0.0.0-20220920034830-41ff9835a66d github.com/go-gost/gosocks4 v0.0.1 github.com/go-gost/gosocks5 v0.3.1-0.20211109033403-d894d75b7f09 github.com/go-gost/relay v0.1.1-0.20211123134818-8ef7fd81ffd7 diff --git a/go.sum b/go.sum index 1548ac6..af51ae0 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,8 @@ github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gost/core v0.0.0-20220914115321-50d443049f3b h1:fWUPYFp0W/6GEhL0wrURGPQN2AQHhf4IZKiALJJOJh8= -github.com/go-gost/core v0.0.0-20220914115321-50d443049f3b/go.mod h1:bHVbCS9da6XtKNYMkMUVcck5UqDDUkyC37erVfs4GXQ= +github.com/go-gost/core v0.0.0-20220920034830-41ff9835a66d h1:UFn21xIJgWE/te12rzQA7Ymwbo+MaxOcp38K41L+Yck= +github.com/go-gost/core v0.0.0-20220920034830-41ff9835a66d/go.mod h1:bHVbCS9da6XtKNYMkMUVcck5UqDDUkyC37erVfs4GXQ= github.com/go-gost/gosocks4 v0.0.1 h1:+k1sec8HlELuQV7rWftIkmy8UijzUt2I6t+iMPlGB2s= github.com/go-gost/gosocks4 v0.0.1/go.mod h1:3B6L47HbU/qugDg4JnoFPHgJXE43Inz8Bah1QaN9qCc= github.com/go-gost/gosocks5 v0.3.1-0.20211109033403-d894d75b7f09 h1:A95M6UWcfZgOuJkQ7QLfG0Hs5peWIUSysCDNz4pfe04= diff --git a/hosts/hosts.go b/hosts/hosts.go index 83af5fe..99c8c17 100644 --- a/hosts/hosts.go +++ b/hosts/hosts.go @@ -23,6 +23,7 @@ type options struct { mappings []Mapping fileLoader loader.Loader redisLoader loader.Loader + httpLoader loader.Loader period time.Duration logger logger.Logger } @@ -53,6 +54,12 @@ func RedisLoaderOption(redisLoader loader.Loader) Option { } } +func HTTPLoaderOption(httpLoader loader.Loader) Option { + return func(opts *options) { + opts.httpLoader = httpLoader + } +} + func LoggerOption(logger logger.Logger) Option { return func(opts *options) { opts.logger = logger @@ -254,6 +261,15 @@ func (h *hostMapper) load(ctx context.Context) (mappings []Mapping, err error) { } } } + if h.options.httpLoader != nil { + r, er := h.options.httpLoader.Load(ctx) + if er != nil { + h.options.logger.Warnf("http loader: %v", er) + } + if m, _ := h.parseMapping(r); m != nil { + mappings = append(mappings, m...) + } + } h.options.logger.Debugf("load items %d", len(mappings)) return diff --git a/internal/loader/http.go b/internal/loader/http.go new file mode 100644 index 0000000..0d9a629 --- /dev/null +++ b/internal/loader/http.go @@ -0,0 +1,71 @@ +package loader + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "time" +) + +type httpLoaderOptions struct { + timeout time.Duration +} + +type HTTPLoaderOption func(opts *httpLoaderOptions) + +func TimeoutHTTPLoaderOption(timeout time.Duration) HTTPLoaderOption { + return func(opts *httpLoaderOptions) { + opts.timeout = timeout + } +} + +type httpLoader struct { + url string + httpClient *http.Client +} + +// HTTPLoader loads data from HTTP request. +func HTTPLoader(url string, opts ...HTTPLoaderOption) Loader { + var options httpLoaderOptions + for _, opt := range opts { + if opt != nil { + opt(&options) + } + } + return &httpLoader{ + url: url, + httpClient: &http.Client{ + Timeout: options.timeout, + }, + } +} + +func (l *httpLoader) Load(ctx context.Context) (io.Reader, error) { + req, err := http.NewRequest(http.MethodGet, l.url, nil) + if err != nil { + return nil, err + } + + resp, err := l.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%d %s", resp.StatusCode, resp.Status) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return bytes.NewReader(data), nil +} + +func (l *httpLoader) Close() error { + return nil +} diff --git a/internal/loader/redis.go b/internal/loader/redis.go index daced47..c19331d 100644 --- a/internal/loader/redis.go +++ b/internal/loader/redis.go @@ -49,7 +49,9 @@ type redisSetLoader struct { func RedisSetLoader(addr string, opts ...RedisLoaderOption) Loader { var options redisLoaderOptions for _, opt := range opts { - opt(&options) + if opt != nil { + opt(&options) + } } key := options.key diff --git a/limiter/conn/conn.go b/limiter/conn/conn.go index 961fc25..6f06dbe 100644 --- a/limiter/conn/conn.go +++ b/limiter/conn/conn.go @@ -62,6 +62,7 @@ type options struct { limits []string fileLoader loader.Loader redisLoader loader.Loader + httpLoader loader.Loader period time.Duration logger logger.Logger } @@ -92,6 +93,12 @@ func RedisLoaderOption(redisLoader loader.Loader) Option { } } +func HTTPLoaderOption(httpLoader loader.Loader) Option { + return func(opts *options) { + opts.httpLoader = httpLoader + } +} + func LoggerOption(logger logger.Logger) Option { return func(opts *options) { opts.logger = logger @@ -292,6 +299,15 @@ func (l *connLimiter) load(ctx context.Context) (patterns []string, err error) { } } } + if l.options.httpLoader != nil { + r, er := l.options.httpLoader.Load(ctx) + if er != nil { + l.options.logger.Warnf("http loader: %v", er) + } + if v, _ := l.parsePatterns(r); v != nil { + patterns = append(patterns, v...) + } + } l.options.logger.Debugf("load items %d", len(patterns)) return diff --git a/limiter/rate/rate.go b/limiter/rate/rate.go index a52ecb8..7483265 100644 --- a/limiter/rate/rate.go +++ b/limiter/rate/rate.go @@ -55,6 +55,7 @@ type options struct { limits []string fileLoader loader.Loader redisLoader loader.Loader + httpLoader loader.Loader period time.Duration logger logger.Logger } @@ -85,6 +86,12 @@ func RedisLoaderOption(redisLoader loader.Loader) Option { } } +func HTTPLoaderOption(httpLoader loader.Loader) Option { + return func(opts *options) { + opts.httpLoader = httpLoader + } +} + func LoggerOption(logger logger.Logger) Option { return func(opts *options) { opts.logger = logger @@ -285,6 +292,15 @@ func (l *rateLimiter) load(ctx context.Context) (patterns []string, err error) { } } } + if l.options.httpLoader != nil { + r, er := l.options.httpLoader.Load(ctx) + if er != nil { + l.options.logger.Warnf("http loader: %v", er) + } + if v, _ := l.parsePatterns(r); v != nil { + patterns = append(patterns, v...) + } + } l.options.logger.Debugf("load items %d", len(patterns)) return diff --git a/limiter/traffic/traffic.go b/limiter/traffic/traffic.go index 54260e2..e42f7b2 100644 --- a/limiter/traffic/traffic.go +++ b/limiter/traffic/traffic.go @@ -54,6 +54,7 @@ type options struct { limits []string fileLoader loader.Loader redisLoader loader.Loader + httpLoader loader.Loader period time.Duration logger logger.Logger } @@ -84,6 +85,12 @@ func RedisLoaderOption(redisLoader loader.Loader) Option { } } +func HTTPLoaderOption(httpLoader loader.Loader) Option { + return func(opts *options) { + opts.httpLoader = httpLoader + } +} + func LoggerOption(logger logger.Logger) Option { return func(opts *options) { opts.logger = logger @@ -337,6 +344,15 @@ func (l *trafficLimiter) load(ctx context.Context) (patterns []string, err error } } } + if l.options.httpLoader != nil { + r, er := l.options.httpLoader.Load(ctx) + if er != nil { + l.options.logger.Warnf("http loader: %v", er) + } + if v, _ := l.parsePatterns(r); v != nil { + patterns = append(patterns, v...) + } + } l.options.logger.Debugf("load items %d", len(patterns)) return