add traffic sniffing for handler

This commit is contained in:
ginuerzh
2024-09-27 15:55:28 +08:00
parent dfb6cb95d0
commit 3c8add4b82
43 changed files with 4518 additions and 1839 deletions
+174 -152
View File
@@ -32,6 +32,7 @@ import (
netpkg "github.com/go-gost/x/internal/net"
xhttp "github.com/go-gost/x/internal/net/http"
limiter_util "github.com/go-gost/x/internal/util/limiter"
"github.com/go-gost/x/internal/util/sniffing"
stats_util "github.com/go-gost/x/internal/util/stats"
rate_limiter "github.com/go-gost/x/limiter/rate"
traffic_wrapper "github.com/go-gost/x/limiter/traffic/wrapper"
@@ -40,10 +41,6 @@ import (
"github.com/go-gost/x/registry"
)
const (
defaultBodySize = 1024 * 1024 // 1MB
)
func init() {
registry.HandlerRegistry().Register("http", NewHandler)
}
@@ -104,6 +101,7 @@ func (h *httpHandler) Handle(ctx context.Context, conn net.Conn, opts ...handler
Service: h.options.Service,
RemoteAddr: conn.RemoteAddr().String(),
LocalAddr: conn.LocalAddr().String(),
Proto: "http",
Time: start,
SID: string(ctxvalue.SidFromContext(ctx)),
}
@@ -116,12 +114,12 @@ func (h *httpHandler) Handle(ctx context.Context, conn net.Conn, opts ...handler
})
log.Infof("%s <> %s", conn.RemoteAddr(), conn.LocalAddr())
defer func() {
if !ro.Time.IsZero() {
if err != nil {
ro.Err = err.Error()
}
ro.Duration = time.Since(start)
ro.Record(ctx, h.recorder.Recorder)
if err != nil {
ro.Err = err.Error()
}
ro.Duration = time.Since(start)
if err := ro.Record(ctx, h.recorder.Recorder); err != nil {
log.Error("record: %v", err)
}
log.WithFields(map[string]any{
@@ -178,11 +176,11 @@ func (h *httpHandler) handleRequest(ctx context.Context, conn net.Conn, req *htt
}
}
ro.Host = req.Host
addr := req.Host
if _, port, _ := net.SplitHostPort(addr); port == "" {
addr = net.JoinHostPort(strings.Trim(addr, "[]"), "80")
}
ro.Host = addr
fields := map[string]any{
"dst": addr,
@@ -248,7 +246,7 @@ func (h *httpHandler) handleRequest(ctx context.Context, conn net.Conn, req *htt
}
if network == "udp" {
return h.handleUDP(ctx, conn, log)
return h.handleUDP(ctx, conn, ro, log)
}
if req.Method == "PRI" ||
@@ -306,8 +304,7 @@ func (h *httpHandler) handleRequest(ctx context.Context, conn net.Conn, req *htt
ro2 := &xrecorder.HandlerRecorderObject{}
*ro2 = *ro
ro.Time = time.Time{}
return h.handleProxy(ctx, xio.NewReadWriteCloser(rw, rw, conn), cc, req, ro2, log)
return h.handleProxy(ctx, rw, cc, req, ro2, log)
}
resp.StatusCode = http.StatusOK
@@ -322,6 +319,31 @@ func (h *httpHandler) handleRequest(ctx context.Context, conn net.Conn, req *htt
return err
}
if h.md.sniffing {
if h.md.sniffingTimeout > 0 {
conn.SetReadDeadline(time.Now().Add(h.md.sniffingTimeout))
}
br := bufio.NewReader(conn)
proto, _ := sniffing.Sniff(ctx, br)
ro.Proto = proto
if h.md.sniffingTimeout > 0 {
conn.SetReadDeadline(time.Time{})
}
rw = xio.NewReadWriter(br, conn)
switch proto {
case sniffing.ProtoHTTP:
ro2 := &xrecorder.HandlerRecorderObject{}
*ro2 = *ro
ro.Time = time.Time{}
return h.handleHTTP(ctx, rw, cc, ro2, log)
case sniffing.ProtoTLS:
return h.handleTLS(ctx, rw, cc, ro, log)
}
}
start := time.Now()
log.Infof("%s <-> %s", conn.RemoteAddr(), addr)
netpkg.Transport(rw, cc)
@@ -332,143 +354,8 @@ func (h *httpHandler) handleRequest(ctx context.Context, conn net.Conn, req *htt
return nil
}
func (h *httpHandler) handleProxy(ctx context.Context, rw io.ReadWriteCloser, cc io.ReadWriter, req *http.Request, ro *xrecorder.HandlerRecorderObject, log logger.Logger) (err error) {
roundTrip := func(req *http.Request) error {
if req == nil {
return nil
}
resp := &http.Response{
ProtoMajor: req.ProtoMajor,
ProtoMinor: req.ProtoMinor,
Header: http.Header{},
StatusCode: http.StatusServiceUnavailable,
}
start := time.Now()
ro.Time = start
ro.HTTP = &xrecorder.HTTPRecorderObject{
Host: req.Host,
Proto: req.Proto,
Scheme: req.URL.Scheme,
Method: req.Method,
URI: req.RequestURI,
StatusCode: resp.StatusCode,
Request: xrecorder.HTTPRequestRecorderObject{
ContentLength: req.ContentLength,
Header: req.Header.Clone(),
},
}
// HTTP/1.0
if req.ProtoMajor == 1 && req.ProtoMinor == 0 {
if strings.ToLower(req.Header.Get("Connection")) == "keep-alive" {
req.Header.Del("Connection")
} else {
req.Header.Set("Connection", "close")
}
}
req.Header.Del("Proxy-Authorization")
req.Header.Del("Proxy-Connection")
req.Header.Del("Gost-Target")
req.Header.Del("X-Gost-Target")
var reqBody *xhttp.Body
if opts := h.recorder.Options; opts != nil && opts.HTTPBody {
if req.Body != nil {
maxSize := opts.MaxBodySize
if maxSize <= 0 {
maxSize = defaultBodySize
}
reqBody = xhttp.NewBody(req.Body, maxSize)
req.Body = reqBody
}
}
if err = req.Write(cc); err != nil {
resp.Write(rw)
ro.Duration = time.Since(start)
ro.Err = err.Error()
ro.Record(ctx, h.recorder.Recorder)
return err
}
if reqBody != nil {
ro.HTTP.Request.Body = reqBody.Content()
ro.HTTP.Request.ContentLength = reqBody.Length()
}
go func() {
var err error
var res *http.Response
var respBody *xhttp.Body
defer func() {
ro.Duration = time.Since(start)
if err != nil {
ro.Err = err.Error()
}
if res != nil {
ro.HTTP.StatusCode = res.StatusCode
ro.HTTP.Response.Header = res.Header
ro.HTTP.Response.ContentLength = res.ContentLength
if respBody != nil {
ro.HTTP.Response.Body = respBody.Content()
ro.HTTP.Response.ContentLength = respBody.Length()
}
}
ro.Record(ctx, h.recorder.Recorder)
}()
res, err = http.ReadResponse(bufio.NewReader(cc), req)
if err != nil {
h.options.Logger.Errorf("read response: %v", err)
resp.Write(rw)
return
}
defer res.Body.Close()
if log.IsLevelEnabled(logger.TraceLevel) {
dump, _ := httputil.DumpResponse(res, false)
log.Trace(string(dump))
}
if res.Close {
defer rw.Close()
}
// HTTP/1.0
if req.ProtoMajor == 1 && req.ProtoMinor == 0 {
if !res.Close {
res.Header.Set("Connection", "keep-alive")
}
res.ProtoMajor = req.ProtoMajor
res.ProtoMinor = req.ProtoMinor
}
if opts := h.recorder.Options; opts != nil && opts.HTTPBody {
maxSize := opts.MaxBodySize
if maxSize <= 0 {
maxSize = defaultBodySize
}
respBody = xhttp.NewBody(res.Body, maxSize)
res.Body = respBody
}
if err = res.Write(rw); err != nil {
rw.Close()
log.Errorf("write response: %v", err)
}
}()
return nil
}
if err = roundTrip(req); err != nil {
func (h *httpHandler) handleProxy(ctx context.Context, rw io.ReadWriter, cc net.Conn, req *http.Request, ro *xrecorder.HandlerRecorderObject, log logger.Logger) error {
if shouldClose, err := h.proxyRoundTrip(ctx, rw, cc, req, ro, log); err != nil || shouldClose {
return err
}
@@ -486,12 +373,147 @@ func (h *httpHandler) handleProxy(ctx context.Context, rw io.ReadWriteCloser, cc
log.Trace(string(dump))
}
if err = roundTrip(req); err != nil {
if shouldClose, err := h.proxyRoundTrip(ctx, rw, cc, req, ro, log); err != nil || shouldClose {
return err
}
}
}
func (h *httpHandler) proxyRoundTrip(ctx context.Context, rw io.ReadWriter, cc net.Conn, req *http.Request, ro *xrecorder.HandlerRecorderObject, log logger.Logger) (close bool, err error) {
close = true
if req == nil {
return
}
start := time.Now()
ro.Time = start
log.Infof("%s <-> %s", ro.RemoteAddr, req.Host)
defer func() {
ro.Duration = time.Since(start)
if err != nil {
ro.Err = err.Error()
}
if err := ro.Record(ctx, h.recorder.Recorder); err != nil {
log.Errorf("record: %v", err)
}
log.WithFields(map[string]any{
"duration": time.Since(start),
}).Infof("%s >-< %s", ro.RemoteAddr, req.Host)
}()
if clientIP := xhttp.GetClientIP(req); clientIP != nil {
ro.ClientIP = clientIP.String()
}
ro.HTTP = &xrecorder.HTTPRecorderObject{
Host: req.Host,
Proto: req.Proto,
Scheme: req.URL.Scheme,
Method: req.Method,
URI: req.RequestURI,
Request: xrecorder.HTTPRequestRecorderObject{
ContentLength: req.ContentLength,
Header: req.Header.Clone(),
},
}
// HTTP/1.0
if req.ProtoMajor == 1 && req.ProtoMinor == 0 {
if strings.ToLower(req.Header.Get("Connection")) == "keep-alive" {
req.Header.Del("Connection")
} else {
req.Header.Set("Connection", "close")
}
}
req.Header.Del("Proxy-Authorization")
req.Header.Del("Proxy-Connection")
req.Header.Del("Gost-Target")
req.Header.Del("X-Gost-Target")
var reqBody *xhttp.Body
if opts := h.recorder.Options; opts != nil && opts.HTTPBody {
if req.Body != nil {
maxSize := opts.MaxBodySize
if maxSize <= 0 {
maxSize = defaultBodySize
}
reqBody = xhttp.NewBody(req.Body, maxSize)
req.Body = reqBody
}
}
res := &http.Response{
ProtoMajor: req.ProtoMajor,
ProtoMinor: req.ProtoMinor,
Header: http.Header{},
StatusCode: http.StatusServiceUnavailable,
}
ro.HTTP.StatusCode = res.StatusCode
if err = req.Write(cc); err != nil {
res.Write(rw)
return
}
if reqBody != nil {
ro.HTTP.Request.Body = reqBody.Content()
ro.HTTP.Request.ContentLength = reqBody.Length()
}
cc.SetReadDeadline(time.Now().Add(h.md.readTimeout))
resp, err := http.ReadResponse(bufio.NewReader(cc), req)
if err != nil {
log.Errorf("read response: %v", err)
res.Write(rw)
return
}
defer resp.Body.Close()
cc.SetReadDeadline(time.Time{})
ro.HTTP.StatusCode = resp.StatusCode
ro.HTTP.Response.Header = resp.Header.Clone()
ro.HTTP.Response.ContentLength = resp.ContentLength
if log.IsLevelEnabled(logger.TraceLevel) {
dump, _ := httputil.DumpResponse(resp, false)
log.Trace(string(dump))
}
// HTTP/1.0
if req.ProtoMajor == 1 && req.ProtoMinor == 0 {
if !resp.Close {
resp.Header.Set("Connection", "keep-alive")
}
resp.ProtoMajor = req.ProtoMajor
resp.ProtoMinor = req.ProtoMinor
}
var respBody *xhttp.Body
if opts := h.recorder.Options; opts != nil && opts.HTTPBody {
maxSize := opts.MaxBodySize
if maxSize <= 0 {
maxSize = defaultBodySize
}
respBody = xhttp.NewBody(resp.Body, maxSize)
resp.Body = respBody
}
if err = resp.Write(rw); err != nil {
log.Errorf("write response: %v", err)
return
}
if respBody != nil {
ro.HTTP.Response.Body = respBody.Content()
ro.HTTP.Response.ContentLength = respBody.Length()
}
return resp.Close, nil
}
func (h *httpHandler) decodeServerName(s string) (string, error) {
b, err := base64.RawURLEncoding.DecodeString(s)
if err != nil {
+11
View File
@@ -15,6 +15,7 @@ const (
)
type metadata struct {
readTimeout time.Duration
probeResistance *probeResistance
enableUDP bool
header http.Header
@@ -22,9 +23,16 @@ type metadata struct {
authBasicRealm string
observePeriod time.Duration
proxyAgent string
sniffing bool
sniffingTimeout time.Duration
}
func (h *httpHandler) parseMetadata(md mdata.Metadata) error {
h.md.readTimeout = mdutil.GetDuration(md, "readTimeout")
if h.md.readTimeout <= 0 {
h.md.readTimeout = 15 * time.Second
}
if m := mdutil.GetStringMapString(md, "http.header", "header"); len(m) > 0 {
hd := http.Header{}
for k, v := range m {
@@ -53,6 +61,9 @@ func (h *httpHandler) parseMetadata(md mdata.Metadata) error {
h.md.proxyAgent = defaultProxyAgent
}
h.md.sniffing = mdutil.GetBool(md, "sniffing")
h.md.sniffingTimeout = mdutil.GetDuration(md, "sniffing.timeout")
return nil
}
+259
View File
@@ -0,0 +1,259 @@
package http
import (
"bufio"
"bytes"
"context"
"encoding/hex"
"errors"
"io"
"net"
"net/http"
"net/http/httputil"
"strings"
"time"
"github.com/go-gost/core/bypass"
"github.com/go-gost/core/logger"
dissector "github.com/go-gost/tls-dissector"
xbypass "github.com/go-gost/x/bypass"
xio "github.com/go-gost/x/internal/io"
netpkg "github.com/go-gost/x/internal/net"
xhttp "github.com/go-gost/x/internal/net/http"
tls_util "github.com/go-gost/x/internal/util/tls"
xrecorder "github.com/go-gost/x/recorder"
)
const (
defaultBodySize = 1024 * 1024 // 1MB
)
func (h *httpHandler) handleHTTP(ctx context.Context, rw, cc io.ReadWriter, ro *xrecorder.HandlerRecorderObject, log logger.Logger) error {
br := bufio.NewReader(rw)
req, err := http.ReadRequest(br)
if err != nil {
return err
}
if log.IsLevelEnabled(logger.TraceLevel) {
dump, _ := httputil.DumpRequest(req, false)
log.Trace(string(dump))
}
host := req.Host
if _, _, err := net.SplitHostPort(host); err != nil {
host = net.JoinHostPort(strings.Trim(host, "[]"), "80")
}
ro.Host = host
log = log.WithFields(map[string]any{
"host": host,
})
if h.options.Bypass != nil &&
h.options.Bypass.Contains(ctx, "tcp", host, bypass.WithPathOption(req.RequestURI)) {
log.Debugf("bypass: %s %s", host, req.RequestURI)
return xbypass.ErrBypass
}
shouldClose, err := h.httpRoundTrip(ctx, rw, cc, req, ro, log)
if err != nil || shouldClose {
return err
}
for {
req, err := http.ReadRequest(br)
if err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
return nil
}
return err
}
if log.IsLevelEnabled(logger.TraceLevel) {
dump, _ := httputil.DumpRequest(req, false)
log.Trace(string(dump))
}
if shouldClose, err := h.httpRoundTrip(ctx, rw, cc, req, ro, log); err != nil || shouldClose {
return err
}
}
}
func (h *httpHandler) httpRoundTrip(ctx context.Context, rw, cc io.ReadWriter, req *http.Request, ro *xrecorder.HandlerRecorderObject, log logger.Logger) (close bool, err error) {
close = true
if req == nil {
return
}
start := time.Now()
ro.Time = start
log.Infof("%s <-> %s", ro.RemoteAddr, req.Host)
defer func() {
ro.Duration = time.Since(start)
if err != nil {
ro.Err = err.Error()
}
if err := ro.Record(ctx, h.recorder.Recorder); err != nil {
log.Errorf("record: %v", err)
}
log.WithFields(map[string]any{
"duration": time.Since(start),
}).Infof("%s >-< %s", ro.RemoteAddr, req.Host)
}()
if clientIP := xhttp.GetClientIP(req); clientIP != nil {
ro.ClientIP = clientIP.String()
}
ro.HTTP = &xrecorder.HTTPRecorderObject{
Host: req.Host,
Proto: req.Proto,
Scheme: req.URL.Scheme,
Method: req.Method,
URI: req.RequestURI,
Request: xrecorder.HTTPRequestRecorderObject{
ContentLength: req.ContentLength,
Header: req.Header.Clone(),
},
}
// HTTP/1.0
if req.ProtoMajor == 1 && req.ProtoMinor == 0 {
if strings.ToLower(req.Header.Get("Connection")) == "keep-alive" {
req.Header.Del("Connection")
} else {
req.Header.Set("Connection", "close")
}
}
var reqBody *xhttp.Body
if opts := h.recorder.Options; opts != nil && opts.HTTPBody {
if req.Body != nil {
maxSize := opts.MaxBodySize
if maxSize <= 0 {
maxSize = defaultBodySize
}
reqBody = xhttp.NewBody(req.Body, maxSize)
req.Body = reqBody
}
}
if err = req.Write(cc); err != nil {
return
}
if reqBody != nil {
ro.HTTP.Request.Body = reqBody.Content()
ro.HTTP.Request.ContentLength = reqBody.Length()
}
xio.SetReadDeadline(cc, time.Now().Add(h.md.readTimeout))
resp, err := http.ReadResponse(bufio.NewReader(cc), req)
if err != nil {
log.Errorf("read response: %v", err)
return
}
defer resp.Body.Close()
xio.SetReadDeadline(cc, time.Time{})
ro.HTTP.StatusCode = resp.StatusCode
ro.HTTP.Response.Header = resp.Header
ro.HTTP.Response.ContentLength = resp.ContentLength
if log.IsLevelEnabled(logger.TraceLevel) {
dump, _ := httputil.DumpResponse(resp, false)
log.Trace(string(dump))
}
// HTTP/1.0
if req.ProtoMajor == 1 && req.ProtoMinor == 0 {
if !resp.Close {
resp.Header.Set("Connection", "keep-alive")
}
resp.ProtoMajor = req.ProtoMajor
resp.ProtoMinor = req.ProtoMinor
}
var respBody *xhttp.Body
if opts := h.recorder.Options; opts != nil && opts.HTTPBody {
maxSize := opts.MaxBodySize
if maxSize <= 0 {
maxSize = defaultBodySize
}
respBody = xhttp.NewBody(resp.Body, maxSize)
resp.Body = respBody
}
if err = resp.Write(rw); err != nil {
log.Errorf("write response: %v", err)
return
}
if respBody != nil {
ro.HTTP.Response.Body = respBody.Content()
ro.HTTP.Response.ContentLength = respBody.Length()
}
if req.Header.Get("Upgrade") == "websocket" {
netpkg.Transport(rw, cc)
}
return resp.Close, nil
}
func (h *httpHandler) handleTLS(_ context.Context, rw, cc io.ReadWriter, ro *xrecorder.HandlerRecorderObject, log logger.Logger) error {
buf := new(bytes.Buffer)
clientHello, err := dissector.ParseClientHello(io.TeeReader(rw, buf))
if err != nil {
return err
}
ro.TLS = &xrecorder.TLSRecorderObject{
ServerName: clientHello.ServerName,
ClientHello: hex.EncodeToString(buf.Bytes()),
}
if len(clientHello.SupportedProtos) > 0 {
ro.TLS.Proto = clientHello.SupportedProtos[0]
}
if _, err := buf.WriteTo(cc); err != nil {
return err
}
xio.SetReadDeadline(cc, time.Now().Add(h.md.readTimeout))
serverHello, err := dissector.ParseServerHello(io.TeeReader(cc, buf))
xio.SetReadDeadline(cc, time.Time{})
if serverHello != nil {
ro.TLS.CipherSuite = tls_util.CipherSuite(serverHello.CipherSuite).String()
ro.TLS.CompressionMethod = serverHello.CompressionMethod
if serverHello.Proto != "" {
ro.TLS.Proto = serverHello.Proto
}
if serverHello.Version > 0 {
ro.TLS.Version = tls_util.Version(serverHello.Version).String()
}
}
if buf.Len() > 0 {
ro.TLS.ServerHello = hex.EncodeToString(buf.Bytes())
}
if _, err := buf.WriteTo(rw); err != nil {
return err
}
t := time.Now()
log.Infof("%s <-> %s", ro.RemoteAddr, ro.Host)
netpkg.Transport(rw, cc)
log.WithFields(map[string]any{
"duration": time.Since(t),
}).Infof("%s >-< %s", ro.RemoteAddr, ro.Host)
return err
}
+7 -2
View File
@@ -1,6 +1,7 @@
package http
import (
"bytes"
"context"
"errors"
"net"
@@ -9,11 +10,13 @@ import (
"time"
"github.com/go-gost/core/logger"
ctxvalue "github.com/go-gost/x/ctx"
"github.com/go-gost/x/internal/net/udp"
"github.com/go-gost/x/internal/util/socks"
xrecorder "github.com/go-gost/x/recorder"
)
func (h *httpHandler) handleUDP(ctx context.Context, conn net.Conn, log logger.Logger) error {
func (h *httpHandler) handleUDP(ctx context.Context, conn net.Conn, ro *xrecorder.HandlerRecorderObject, log logger.Logger) error {
log = log.WithFields(map[string]any{
"cmd": "udp",
})
@@ -51,7 +54,9 @@ func (h *httpHandler) handleUDP(ctx context.Context, conn net.Conn, log logger.L
}
// obtain a udp connection
c, err := h.options.Router.Dial(ctx, "udp", "") // UDP association
var buf bytes.Buffer
c, err := h.options.Router.Dial(ctxvalue.ContextWithBuffer(ctx, &buf), "udp", "") // UDP association
ro.Route = buf.String()
if err != nil {
log.Error(err)
return err