From e583f5b53867dab0b504affe2c1168661be59d90 Mon Sep 17 00:00:00 2001 From: ginuerzh Date: Sat, 20 Jun 2026 20:11:43 +0800 Subject: [PATCH] feat(parser): support HTTP(S) URL config sources with -C flag Add remote config fetching capability to the -C CLI flag, enabling centralized fleet management of gost nodes. - readConfig() now detects http:// and https:// URLs and fetches them via HTTP GET with a 30s timeout and shared client pool - Format auto-detection from Content-Type header (using mime parsing) with fallback to URL path extension (.yaml/.json) - 10 MiB response size limit to prevent memory exhaustion - URL userinfo stripped from error messages for safety - Fix ReadFile to force viper config type from file extension, preventing format detection ambiguity - Fix viper config search path fallback: skip cfg.Load() when -C flags are explicitly provided, avoiding unwanted merges - Tests: URL fetch (YAML/JSON), non-200, oversize, detectFormat, isHTTPURL, sanitizeURL, and combined file+URL Parse scenario --- config/config.go | 3 + config/parsing/parser/parse_test.go | 193 ++++++++++++++++++++++++++++ config/parsing/parser/parser.go | 105 ++++++++++++++- 3 files changed, 297 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index fee91d51..7cb19761 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,8 @@ package config import ( "encoding/json" "io" + "path/filepath" + "strings" "sync" "time" @@ -658,6 +660,7 @@ func (c *Config) Read(r io.Reader, configType string) error { func (c *Config) ReadFile(file string) error { v.SetConfigFile(file) + v.SetConfigType(strings.TrimPrefix(filepath.Ext(file), ".")) // force format from extension if err := v.ReadInConfig(); err != nil { return err } diff --git a/config/parsing/parser/parse_test.go b/config/parsing/parser/parse_test.go index 5e157ce2..b72a86f2 100644 --- a/config/parsing/parser/parse_test.go +++ b/config/parsing/parser/parse_test.go @@ -1,6 +1,9 @@ package parser import ( + "bytes" + "net/http" + "net/http/httptest" "os" "path/filepath" "testing" @@ -285,3 +288,193 @@ func TestParse_MultiFile(t *testing.T) { t.Fatal("Quotas not merged from cfg2") } } + +func TestIsHTTPURL(t *testing.T) { + tests := []struct { + input string + expected bool + }{ + {"http://example.com/config.yaml", true}, + {"https://example.com/config.json", true}, + {"ftp://example.com/config.yaml", false}, + {"config.yaml", false}, + {"/path/to/config.yml", false}, + {"http://[::1]:8080/cfg", true}, + } + for _, tt := range tests { + if got := isHTTPURL(tt.input); got != tt.expected { + t.Errorf("isHTTPURL(%q) = %v, want %v", tt.input, got, tt.expected) + } + } +} + +func TestDetectFormat(t *testing.T) { + tests := []struct { + contentType string + url string + want string + }{ + {"", "http://example.com/config.json", "json"}, + {"", "http://example.com/config.yml", "yaml"}, + {"", "http://example.com/config.yaml", "yaml"}, + {"", "http://example.com/config", "yaml"}, // default + {"application/json", "http://example.com/config", "json"}, + {"application/yaml", "http://example.com/config", "yaml"}, + {"application/x-yaml", "http://example.com/config", "yaml"}, + {"text/yaml", "http://example.com/config", "yaml"}, + {"text/x-yaml", "http://example.com/config", "yaml"}, + {"text/plain; charset=utf-8", "http://example.com/config.json", "json"}, // content-type ignored, extension wins + {"text/plain", "http://example.com/config", "yaml"}, // unknown -> default + } + for _, tt := range tests { + if got := detectFormat(tt.contentType, tt.url); got != tt.want { + t.Errorf("detectFormat(%q, %q) = %q, want %q", tt.contentType, tt.url, got, tt.want) + } + } +} + +func TestSanitizeURL(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"http://user:pass@example.com/path", "http://example.com/path"}, + {"https://example.com/path", "https://example.com/path"}, + {"not a url", "not%20a%20url"}, + {"http://user@example.com/path", "http://example.com/path"}, // user with no password + } + for _, tt := range tests { + if got := sanitizeURL(tt.input); got != tt.expected { + t.Errorf("sanitizeURL(%q) = %q, want %q", tt.input, got, tt.expected) + } + } +} + +func TestReadConfigFromURL_YAML(t *testing.T) { + // Write a YAML config and serve it via httptest. + cfg := &config.Config{ + Services: []*config.ServiceConfig{{Name: "svc", Addr: ":8080"}}, + Log: &config.LogConfig{Level: "debug"}, + } + var buf bytes.Buffer + if err := cfg.Write(&buf, "yaml"); err != nil { + t.Fatal(err) + } + body := buf.Bytes() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/yaml") + w.Write(body) + })) + defer ts.Close() + + got := &config.Config{} + if err := readConfigFromURL(ts.URL, got); err != nil { + t.Fatalf("readConfigFromURL: %v", err) + } + if len(got.Services) != 1 || got.Services[0].Name != "svc" { + t.Fatalf("unexpected services: %+v", got.Services) + } + if got.Log == nil || got.Log.Level != "debug" { + t.Fatalf("unexpected log: %+v", got.Log) + } +} + +func TestReadConfigFromURL_JSON(t *testing.T) { + // Remote serving JSON with explicit Content-Type. + cfg := &config.Config{ + Services: []*config.ServiceConfig{{Name: "json-svc", Addr: ":9090"}}, + } + var buf bytes.Buffer + if err := cfg.Write(&buf, "json"); err != nil { + t.Fatal(err) + } + body := buf.Bytes() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write(body) + })) + defer ts.Close() + + got := &config.Config{} + if err := readConfigFromURL(ts.URL, got); err != nil { + t.Fatalf("readConfigFromURL: %v", err) + } + if got.Services[0].Name != "json-svc" { + t.Fatalf("unexpected service name: %s", got.Services[0].Name) + } +} + +func TestReadConfigFromURL_Non200(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer ts.Close() + + err := readConfigFromURL(ts.URL, &config.Config{}) + if err == nil { + t.Fatal("expected error for 404 response") + } +} + +func TestReadConfigFromURL_OverSize(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/yaml") + w.Write(make([]byte, maxConfigSize+1)) + })) + defer ts.Close() + + err := readConfigFromURL(ts.URL, &config.Config{}) + if err == nil { + t.Fatal("expected error for oversized response") + } +} + +func TestParse_FileAndURL(t *testing.T) { + // Create a local file. + fileCfg := &config.Config{ + Services: []*config.ServiceConfig{{Name: "file-svc", Addr: ":8080"}}, + } + tmpDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, "local.yml")) + if err != nil { + t.Fatal(err) + } + if err := fileCfg.Write(f, "yaml"); err != nil { + f.Close() + t.Fatal(err) + } + f.Close() + + // Serve a second config over HTTP. + urlCfg := &config.Config{ + Services: []*config.ServiceConfig{{Name: "url-svc", Addr: ":8081"}}, + Log: &config.LogConfig{Level: "debug"}, + } + var buf bytes.Buffer + if err := urlCfg.Write(&buf, "yaml"); err != nil { + t.Fatal(err) + } + body := buf.Bytes() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/yaml") + w.Write(body) + })) + defer ts.Close() + + Init(Args{ + CfgFiles: []string{filepath.Join(tmpDir, "local.yml"), ts.URL}, + }) + + got, err := Parse() + if err != nil { + t.Fatalf("Parse(): %v", err) + } + if len(got.Services) != 2 { + t.Fatalf("expected 2 services, got %d", len(got.Services)) + } + if got.Log == nil || got.Log.Level != "debug" { + t.Fatal("Log.Level not set from URL config") + } +} diff --git a/config/parsing/parser/parser.go b/config/parsing/parser/parser.go index 3e4c25b3..525ae6c2 100644 --- a/config/parsing/parser/parser.go +++ b/config/parsing/parser/parser.go @@ -2,9 +2,17 @@ package parser import ( "bufio" + "bytes" "encoding/json" + "fmt" + "io" + "mime" + "net/http" + "net/url" "os" + "path" "strings" + "time" "github.com/go-gost/core/logger" "github.com/go-gost/x/config" @@ -35,8 +43,9 @@ func Parse() (*config.Config, error) { // Args holds the raw CLI flags and positional arguments that Init will merge // into the final configuration. type Args struct { - // CfgFiles is the list of YAML or JSON config file paths, "-" for stdin, or - // an inline JSON string. Multiple files are merged left-to-right. + // CfgFiles is the list of YAML or JSON config sources: file paths, + // HTTP(S) URLs, "-" for stdin, or an inline JSON string. + // Multiple sources are merged left-to-right. CfgFiles []string // Services is the list of service definitions from -L flags. @@ -83,7 +92,8 @@ func (p *parser) Parse() (*config.Config, error) { } cfg = mergeConfig(cfg, cmdCfg) - if len(cfg.Services) == 0 && p.args.ApiAddr == "" && cfg.API == nil { + if len(cfg.Services) == 0 && p.args.ApiAddr == "" && cfg.API == nil && + len(p.args.CfgFiles) == 0 { if err := cfg.Load(); err != nil { return nil, err } @@ -179,7 +189,7 @@ func (p *parser) Parse() (*config.Config, error) { } // readConfig reads a single configuration source, which may be "-" (stdin), -// an inline JSON string starting with "{", or a file path. +// an HTTP(S) URL, an inline JSON string starting with "{", or a file path. func readConfig(cfgFile string) (*config.Config, error) { cfg := &config.Config{} @@ -202,6 +212,10 @@ func readConfig(cfgFile string) (*config.Config, error) { if err := json.Unmarshal([]byte(cfgFile), cfg); err != nil { return nil, err } + } else if isHTTPURL(cfgFile) { // URL + if err := readConfigFromURL(cfgFile, cfg); err != nil { + return nil, err + } } else { if err := cfg.ReadFile(cfgFile); err != nil { // file return nil, err @@ -211,6 +225,89 @@ func readConfig(cfgFile string) (*config.Config, error) { return cfg, nil } +// isHTTPURL reports whether s is an HTTP or HTTPS URL. +func isHTTPURL(s string) bool { + u, err := url.Parse(s) + return err == nil && u.Scheme != "" && u.Host != "" && + (u.Scheme == "http" || u.Scheme == "https") +} + +// maxConfigSize is the maximum response body size for remote config files. +const maxConfigSize = 10 << 20 // 10 MiB + +// sharedHTTPClient is reused across config URL fetches to benefit from +// connection pooling. +var sharedHTTPClient = &http.Client{ + Timeout: 30 * time.Second, +} + +// readConfigFromURL fetches a configuration from an HTTP(S) URL and reads it +// into cfg. The format (yaml or json) is detected from the Content-Type header +// or the URL path extension. +func readConfigFromURL(urlStr string, cfg *config.Config) error { + resp, err := sharedHTTPClient.Get(urlStr) + if err != nil { + return fmt.Errorf("fetch config from %s: %w", sanitizeURL(urlStr), err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + io.Copy(io.Discard, resp.Body) // drain body for connection reuse; ignore error since we are already failing + return fmt.Errorf("fetch config from %s: %s", sanitizeURL(urlStr), resp.Status) + } + + data, err := io.ReadAll(io.LimitReader(resp.Body, maxConfigSize+1)) + if err != nil { + return fmt.Errorf("read config from %s: %w", sanitizeURL(urlStr), err) + } + if len(data) > maxConfigSize { + return fmt.Errorf("config from %s exceeds maximum size %d", sanitizeURL(urlStr), maxConfigSize) + } + + format := detectFormat(resp.Header.Get("Content-Type"), urlStr) + + if err := cfg.Read(bytes.NewReader(data), format); err != nil { + return fmt.Errorf("parse config from %s: %w", sanitizeURL(urlStr), err) + } + return nil +} + +// sanitizeURL returns the URL with any embedded userinfo stripped for safe +// use in error messages. +func sanitizeURL(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + u.User = nil + return u.String() +} + +// detectFormat determines the config format from the HTTP Content-Type header +// or the URL path extension. Returns "yaml" or "json". +func detectFormat(contentType, urlStr string) string { + if mediatype, _, err := mime.ParseMediaType(contentType); err == nil { + switch mediatype { + case "application/json": + return "json" + case "application/yaml", "application/x-yaml", + "text/yaml", "text/x-yaml": + return "yaml" + } + } + + if u, err := url.Parse(urlStr); err == nil { + switch strings.ToLower(strings.TrimPrefix(path.Ext(u.Path), ".")) { + case "json": + return "json" + case "yaml", "yml": + return "yaml" + } + } + + return "yaml" +} + func mergeConfig(cfg1, cfg2 *config.Config) *config.Config { if cfg1 == nil { return cfg2