go-gost/gost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve http handler

Browse Source
This commit is contained in:
ginuerzh 2021-10-31 12:41:53 +08:00
parent 248f7e4318
commit 64736585ee
11 changed files with 435 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

224
cmd/gost/config.go
View File

@ -16,6 +16,121 @@ import (
"github.com/go-gost/gost/pkg/service"
)
func buildService(cfg *config.Config) (services []*service.Service) {
if cfg == nil || len(cfg.Services) == 0 {
return
}
chains := buildChain(cfg)
for _, svc := range cfg.Services {
listenerLogger := log.WithFields(map[string]interface{}{
"kind": "listener",
"type": svc.Listener.Type,
"service": svc.Name,
})
ln := registry.GetListener(svc.Listener.Type)(
listener.AddrOption(svc.Addr),
listener.LoggerOption(listenerLogger),
)
if err := ln.Init(metadata.MapMetadata(svc.Listener.Metadata)); err != nil {
listenerLogger.Fatal("init:", err)
}
var chain *chain.Chain
for _, ch := range chains {
if svc.Chain == ch.Name {
chain = ch
break
}
}
handlerLogger := log.WithFields(map[string]interface{}{
"kind": "handler",
"type": svc.Handler.Type,
"service": svc.Name,
})
h := registry.GetHandler(svc.Handler.Type)(
handler.ChainOption(chain),
handler.LoggerOption(handlerLogger),
)
if err := h.Init(metadata.MapMetadata(svc.Handler.Metadata)); err != nil {
handlerLogger.Fatal("init:", err)
}
s := (&service.Service{}).
WithListener(ln).
WithHandler(h)
services = append(services, s)
}
return
}
func buildChain(cfg *config.Config) (chains []*chain.Chain) {
if cfg == nil || len(cfg.Chains) == 0 {
return nil
}
for _, ch := range cfg.Chains {
c := &chain.Chain{
Name: ch.Name,
}
selector := selectorFromConfig(ch.LB)
for _, hop := range ch.Hops {
group := &chain.NodeGroup{}
for _, v := range hop.Nodes {
node := chain.NewNode(v.Name, v.Addr)
connectorLogger := log.WithFields(map[string]interface{}{
"kind": "connector",
"type": v.Connector.Type,
"hop": hop.Name,
"node": node.Name(),
})
cr := registry.GetConnector(v.Connector.Type)(
connector.LoggerOption(connectorLogger),
)
if err := cr.Init(metadata.MapMetadata(v.Connector.Metadata)); err != nil {
connectorLogger.Fatal("init:", err)
}
dialerLogger := log.WithFields(map[string]interface{}{
"kind": "dialer",
"type": v.Dialer.Type,
"hop": hop.Name,
"node": node.Name(),
})
d := registry.GetDialer(v.Dialer.Type)(
dialer.LoggerOption(dialerLogger),
)
if err := d.Init(metadata.MapMetadata(v.Dialer.Metadata)); err != nil {
dialerLogger.Fatal("init:", err)
}
tr := (&chain.Transport{}).
WithConnector(cr).
WithDialer(d)
node.WithTransport(tr)
group.AddNode(node)
}
sel := selector
if s := selectorFromConfig(hop.LB); s != nil {
sel = s
}
group.WithSelector(sel)
c.AddNodeGroup(group)
}
chains = append(chains, c)
}
return
}
func logFromConfig(cfg *config.LogConfig) logger.Logger {
opts := []logger.LoggerOption{
logger.FormatLoggerOption(logger.LogFormat(cfg.Format)),
@ -41,100 +156,29 @@ func logFromConfig(cfg *config.LogConfig) logger.Logger {
return logger.NewLogger(opts...)
}
func buildService(cfg *config.Config) (services []*service.Service) {
if cfg == nil || len(cfg.Services) == 0 {
return
}
chains := buildChain(cfg)
for _, svc := range cfg.Services {
s := &service.Service{}
ln := registry.GetListener(svc.Listener.Type)(
listener.AddrOption(svc.Addr),
listener.LoggerOption(
log.WithFields(map[string]interface{}{
"kind": "listener",
"type": svc.Listener.Type,
}),
),
)
ln.Init(metadata.MapMetadata(svc.Listener.Metadata))
s.WithListener(ln)
var chain *chain.Chain
for _, ch := range chains {
if svc.Chain == ch.Name {
chain = ch
break
}
}
h := registry.GetHandler(svc.Handler.Type)(
handler.ChainOption(chain),
handler.LoggerOption(
log.WithFields(map[string]interface{}{
"kind": "handler",
"type": svc.Handler.Type,
}),
),
)
h.Init(metadata.MapMetadata(svc.Handler.Metadata))
s.WithHandler(h)
services = append(services, s)
}
return
}
func buildChain(cfg *config.Config) (chains []*chain.Chain) {
if cfg == nil || len(cfg.Chains) == 0 {
func selectorFromConfig(cfg *config.LoadbalancingConfig) chain.Selector {
if cfg == nil {
return nil
}
for _, ch := range cfg.Chains {
c := &chain.Chain{
Name: ch.Name,
var strategy chain.Strategy
switch cfg.Strategy {
case "round":
strategy = &chain.RoundRobinStrategy{}
case "random":
strategy = &chain.RandomStrategy{}
case "fifio":
strategy = &chain.FIFOStrategy{}
default:
strategy = &chain.RoundRobinStrategy{}
}
for _, hop := range ch.Hops {
group := &chain.NodeGroup{}
for _, v := range hop.Nodes {
node := chain.NewNode(v.Name, v.Addr)
tr := &chain.Transport{}
cr := registry.GetConnector(v.Connector.Type)(
connector.LoggerOption(
log.WithFields(map[string]interface{}{
"kind": "connector",
"type": v.Connector.Type,
}),
),
return chain.NewSelector(
strategy,
&chain.InvalidFilter{},
&chain.FailFilter{
MaxFails: cfg.MaxFails,
FailTimeout: cfg.FailTimeout,
},
)
cr.Init(metadata.MapMetadata(v.Connector.Metadata))
tr.WithConnector(cr)
d := registry.GetDialer(v.Dialer.Type)(
dialer.LoggerOption(
log.WithFields(map[string]interface{}{
"kind": "dialer",
"type": v.Dialer.Type,
}),
),
)
d.Init(metadata.MapMetadata(v.Dialer.Metadata))
tr.WithDialer(d)
node.WithTransport(tr)
group.AddNode(node)
}
c.AddNodeGroup(group)
}
chains = append(chains, c)
}
return
}

57
cmd/gost/gost.yml
View File

@ -4,46 +4,77 @@ log:
format: json # text, json
services:
- url: "http://gost:gost@:8000"
- name: http+tcp
url: "http://gost:gost@:8000"
addr: ":8000"
handler:
type: http
metadata:
proxyAgent: "gost/3.0"
retry: 3
auths:
- user1:pass1
- user2:pass2
# probeResist: code:404 # code, web, host, file
# knock: example.com
listener:
type: tcp
metadata:
keepAlive: 15s
# chain: chain01
chain: chain01
chains:
- name: chain01
# chain level load balancing
lb:
strategy: round
filters:
- filter1
maxFails: 1
failTimeout: 30s
hops:
- name: level01
- name: hop01
# hop level load balancing
lb:
strategy: rand
filters:
- filter1
strategy: round
maxFails: 1
failTimeout: 30s
nodes:
- name: node01
addr: ":8080"
url: "http://gost:gost@:8080"
addr: ":8081"
url: "http://gost:gost@:8081"
connector:
type: http
metadata:
userAgent: "gost/3.0"
auth: username:password
auth: user1:pass1
dialer:
type: tcp
metadata: {}
- name: node02
addr: ":8082"
url: "http://gost:gost@:8082"
connector:
type: http
metadata:
userAgent: "gost/3.0"
auth: user1:pass1
dialer:
type: tcp
metadata: {}
- name: hop02
# hop level load balancing
lb:
strategy: round
maxFails: 1
failTimeout: 30s
nodes:
- name: node03
addr: ":8083"
url: "http://gost:gost@:8083"
connector:
type: http
metadata:
userAgent: "gost/3.0"
auth: user1:pass1
dialer:
type: tcp
metadata: {}

3
cmd/gost/main.go
View File

@ -1,6 +1,8 @@
package main
import (
stdlog "log"
"github.com/go-gost/gost/pkg/config"
"github.com/go-gost/gost/pkg/logger"
)
@ -10,6 +12,7 @@ var (
)
func main() {
stdlog.SetFlags(stdlog.LstdFlags | stdlog.Lshortfile)
cfg := &config.Config{}
if err := cfg.Load(); err != nil {
log.Fatal(err)

68
pkg/chain/node.go
View File

@ -1,15 +1,22 @@
package chain
import (
"sync"
"time"
)
type Node struct {
name string
addr string
transport *Transport
marker *failMarker
}
func NewNode(name, addr string) *Node {
return &Node{
name: name,
addr: addr,
marker: &failMarker{},
}
}
@ -45,15 +52,72 @@ func (g *NodeGroup) AddNode(node *Node) {
g.nodes = append(g.nodes, node)
}
func (g *NodeGroup) WithSelector(selector Selector) {
func (g *NodeGroup) WithSelector(selector Selector) *NodeGroup {
g.selector = selector
return g
}
func (g *NodeGroup) Next() *Node {
if g == nil || len(g.nodes) == 0 {
return nil
}
selector := g.selector
if selector == nil {
// selector = defaultSelector
return g.nodes[0]
}
return selector.Select(g.nodes...)
}
type failMarker struct {
failTime int64
failCount uint32
mux sync.RWMutex
}
func (m *failMarker) FailTime() int64 {
if m == nil {
return 0
}
m.mux.RLock()
defer m.mux.RUnlock()
return m.failTime
}
func (m *failMarker) FailCount() uint32 {
if m == nil {
return 0
}
m.mux.RLock()
defer m.mux.RUnlock()
return m.failCount
}
func (m *failMarker) Mark() {
if m == nil {
return
}
m.mux.Lock()
defer m.mux.Unlock()
m.failTime = time.Now().Unix()
m.failCount++
}
func (m *failMarker) Reset() {
if m == nil {
return
}
m.mux.Lock()
defer m.mux.Unlock()
m.failTime = 0
m.failCount = 0
}

8
pkg/chain/route.go
View File

@ -22,26 +22,34 @@ func (r *Route) Connect(ctx context.Context) (conn net.Conn, err error) {
node := r.nodes[0]
cc, err := node.Transport().Dial(ctx, r.nodes[0].Addr())
if err != nil {
node.marker.Mark()
return
}
cn, err := node.Transport().Handshake(ctx, cc)
if err != nil {
cc.Close()
node.marker.Mark()
return
}
node.marker.Reset()
preNode := node
for _, node := range r.nodes[1:] {
cc, err = preNode.Transport().Connect(ctx, cn, "tcp", node.Addr())
if err != nil {
cn.Close()
node.marker.Mark()
return
}
cc, err = node.transport.Handshake(ctx, cc)
if err != nil {
cn.Close()
node.marker.Mark()
return
}
node.marker.Reset()
cn = cc
preNode = node
}

137
pkg/chain/selector.go
View File

@ -1,19 +1,24 @@
package chain
import (
"math/rand"
"net"
"strconv"
"sync"
"sync/atomic"
"time"
)
// default options for FailFilter
const (
DefaultMaxFails = 1
DefaultFailTimeout = 30 * time.Second
)
var (
defaultSelector Selector = NewSelector(nil)
)
type Filter interface {
Filter(nodes ...*Node) []*Node
String() string
}
type Strategy interface {
Apply(nodes ...*Node) *Node
String() string
}
type Selector interface {
Select(nodes ...*Node) *Node
}
@ -39,3 +44,115 @@ func (s *selector) Select(nodes ...*Node) *Node {
}
return s.strategy.Apply(nodes...)
}
type Strategy interface {
Apply(nodes ...*Node) *Node
}
// RoundStrategy is a strategy for node selector.
// The node will be selected by round-robin algorithm.
type RoundRobinStrategy struct {
counter uint64
}
func (s *RoundRobinStrategy) Apply(nodes ...*Node) *Node {
if len(nodes) == 0 {
return nil
}
n := atomic.AddUint64(&s.counter, 1) - 1
return nodes[int(n%uint64(len(nodes)))]
}
// RandomStrategy is a strategy for node selector.
// The node will be selected randomly.
type RandomStrategy struct {
Seed int64
rand *rand.Rand
once sync.Once
mux sync.Mutex
}
func (s *RandomStrategy) Apply(nodes ...*Node) *Node {
s.once.Do(func() {
seed := s.Seed
if seed == 0 {
seed = time.Now().UnixNano()
}
s.rand = rand.New(rand.NewSource(seed))
})
if len(nodes) == 0 {
return nil
}
s.mux.Lock()
defer s.mux.Unlock()
r := s.rand.Int()
return nodes[r%len(nodes)]
}
// FIFOStrategy is a strategy for node selector.
// The node will be selected from first to last,
// and will stick to the selected node until it is failed.
type FIFOStrategy struct{}
// Apply applies the fifo strategy for the nodes.
func (s *FIFOStrategy) Apply(nodes ...*Node) *Node {
if len(nodes) == 0 {
return nil
}
return nodes[0]
}
type Filter interface {
Filter(nodes ...*Node) []*Node
}
// FailFilter filters the dead node.
// A node is marked as dead if its failed count is greater than MaxFails.
type FailFilter struct {
MaxFails int
FailTimeout time.Duration
}
// Filter filters dead nodes.
func (f *FailFilter) Filter(nodes ...*Node) []*Node {
maxFails := f.MaxFails
if maxFails == 0 {
maxFails = DefaultMaxFails
}
failTimeout := f.FailTimeout
if failTimeout == 0 {
failTimeout = DefaultFailTimeout
}
if len(nodes) <= 1 || maxFails < 0 {
return nodes
}
var nl []*Node
for _, node := range nodes {
if node.marker.FailCount() < uint32(maxFails) ||
time.Since(time.Unix(node.marker.FailTime(), 0)) >= failTimeout {
nl = append(nl, node)
}
}
return nl
}
// InvalidFilter filters the invalid node.
// A node is invalid if its port is invalid (negative or zero value).
type InvalidFilter struct{}
// Filter filters invalid nodes.
func (f *InvalidFilter) Filter(nodes ...*Node) []*Node {
var nl []*Node
for _, node := range nodes {
_, sport, _ := net.SplitHostPort(node.Addr())
if port, _ := strconv.Atoi(sport); port > 0 {
nl = append(nl, node)
}
}
return nl
}

18
pkg/components/connector/http/connector.go
View File

@ -5,9 +5,9 @@ import (
"context"
"encoding/base64"
"fmt"
"log"
"net"
"net/http"
"net/http/httputil"
"net/url"
"strings"
@ -51,11 +51,15 @@ func (c *Connector) Connect(ctx context.Context, conn net.Conn, network, address
Header: make(http.Header),
}
if c.md.UserAgent != "" {
log.Println(c.md.UserAgent)
req.Header.Set("User-Agent", c.md.UserAgent)
}
req.Header.Set("Proxy-Connection", "keep-alive")
c.logger = c.logger.WithFields(map[string]interface{}{
"src": conn.LocalAddr().String(),
"dst": conn.RemoteAddr().String(),
})
if user := c.md.User; user != nil {
u := user.Username()
p, _ := user.Password()
@ -63,6 +67,11 @@ func (c *Connector) Connect(ctx context.Context, conn net.Conn, network, address
"Basic "+base64.StdEncoding.EncodeToString([]byte(u+":"+p)))
}
if c.logger.IsLevelEnabled(logger.DebugLevel) {
dump, _ := httputil.DumpRequest(req, false)
c.logger.Debug(string(dump))
}
req = req.WithContext(ctx)
if err := req.Write(conn); err != nil {
return nil, err
@ -74,6 +83,11 @@ func (c *Connector) Connect(ctx context.Context, conn net.Conn, network, address
}
defer resp.Body.Close()
if c.logger.IsLevelEnabled(logger.DebugLevel) {
dump, _ := httputil.DumpResponse(resp, false)
c.logger.Debug(string(dump))
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%s", resp.Status)
}

26
pkg/components/dialer/tcp/dialer.go
View File

@ -42,11 +42,33 @@ func (d *Dialer) Dial(ctx context.Context, addr string, opts ...dialer.DialOptio
dial := options.DialFunc
if dial != nil {
return dial(ctx, addr)
conn, err := dial(ctx, addr)
if err != nil {
d.logger.Error(err)
} else {
if d.logger.IsLevelEnabled(logger.DebugLevel) {
d.logger.WithFields(map[string]interface{}{
"src": conn.LocalAddr().String(),
"dst": addr,
}).Debug("dial with dial func")
}
}
return conn, err
}
var netd net.Dialer
return netd.DialContext(ctx, "tcp", addr)
conn, err := netd.DialContext(ctx, "tcp", addr)
if err != nil {
d.logger.Error(err)
} else {
if d.logger.IsLevelEnabled(logger.DebugLevel) {
d.logger.WithFields(map[string]interface{}{
"src": conn.LocalAddr().String(),
"dst": addr,
}).Debug("dial direct")
}
}
return conn, err
}
func (d *Dialer) parseMetadata(md md.Metadata) (err error) {

7
pkg/components/handler/http/handler.go
View File

@ -76,6 +76,7 @@ func (h *Handler) parseMetadata(md md.Metadata) error {
}
}
}
h.md.retryCount = md.GetInt(retryCount)
return nil
}
@ -260,10 +261,10 @@ func (h *Handler) dial(ctx context.Context, addr string) (conn net.Conn, err err
*/
conn, err = route.Dial(ctx, "tcp", addr)
if err != nil {
h.logger.Warn("retry:", err)
continue
if err == nil {
break
}
h.logger.Errorf("route(retry=%d): %s", i, err)
}
return

1
pkg/components/handler/http/metadata.go
View File

@ -8,6 +8,7 @@ const (
authsKey = "auths"
probeResistKey = "probeResist"
knockKey = "knock"
retryCount = "retry"
)
type metadata struct {

5
pkg/config/config.go
View File

@ -2,6 +2,7 @@ package config
import (
"io"
"time"
"github.com/spf13/viper"
)
@ -25,7 +26,8 @@ type LogConfig struct {
type LoadbalancingConfig struct {
Strategy string
Filters []string
MaxFails int
FailTimeout time.Duration
}
type ListenerConfig struct {
@ -49,6 +51,7 @@ type ConnectorConfig struct {
}
type ServiceConfig struct {
Name string
URL string
Addr string
Listener *ListenerConfig