feat(parser): support HTTP(S) URL config sources with -C flag

Add remote config fetching capability to the -C CLI flag, enabling centralized fleet management of gost nodes. - readConfig() now detects http:// and https:// URLs and fetches them via HTTP GET with a 30s timeout and shared client pool - Format auto-detection from Content-Type header (using mime parsing) with fallback to URL path extension (.yaml/.json) - 10 MiB response size limit to prevent memory exhaustion - URL userinfo stripped from error messages for safety - Fix ReadFile to force viper config type from file extension, preventing format detection ambiguity - Fix viper config search path fallback: skip cfg.Load() when -C flags are explicitly provided, avoiding unwanted merges - Tests: URL fetch (YAML/JSON), non-200, oversize, detectFormat, isHTTPURL, sanitizeURL, and combined file+URL Parse scenario
2026-06-20 20:11:43 +08:00
parent 9c585e2de8
commit e583f5b538
3 changed files with 297 additions and 4 deletions
@@ -3,6 +3,8 @@ package config
 import (
 	"encoding/json"
 	"io"
+	"path/filepath"
+	"strings"
 	"sync"
 	"time"

@@ -658,6 +660,7 @@ func (c *Config) Read(r io.Reader, configType string) error {

 func (c *Config) ReadFile(file string) error {
 	v.SetConfigFile(file)
+	v.SetConfigType(strings.TrimPrefix(filepath.Ext(file), ".")) // force format from extension
 	if err := v.ReadInConfig(); err != nil {
 		return err
 	}
@@ -1,6 +1,9 @@
 package parser

 import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"testing"
@@ -285,3 +288,193 @@ func TestParse_MultiFile(t *testing.T) {
 		t.Fatal("Quotas not merged from cfg2")
 	}
 }
+
+func TestIsHTTPURL(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected bool
+	}{
+		{"http://example.com/config.yaml", true},
+		{"https://example.com/config.json", true},
+		{"ftp://example.com/config.yaml", false},
+		{"config.yaml", false},
+		{"/path/to/config.yml", false},
+		{"http://[::1]:8080/cfg", true},
+	}
+	for _, tt := range tests {
+		if got := isHTTPURL(tt.input); got != tt.expected {
+			t.Errorf("isHTTPURL(%q) = %v, want %v", tt.input, got, tt.expected)
+		}
+	}
+}
+
+func TestDetectFormat(t *testing.T) {
+	tests := []struct {
+		contentType string
+		url         string
+		want        string
+	}{
+		{"", "http://example.com/config.json", "json"},
+		{"", "http://example.com/config.yml", "yaml"},
+		{"", "http://example.com/config.yaml", "yaml"},
+		{"", "http://example.com/config", "yaml"}, // default
+		{"application/json", "http://example.com/config", "json"},
+		{"application/yaml", "http://example.com/config", "yaml"},
+		{"application/x-yaml", "http://example.com/config", "yaml"},
+		{"text/yaml", "http://example.com/config", "yaml"},
+		{"text/x-yaml", "http://example.com/config", "yaml"},
+		{"text/plain; charset=utf-8", "http://example.com/config.json", "json"}, // content-type ignored, extension wins
+		{"text/plain", "http://example.com/config", "yaml"},                     // unknown -> default
+	}
+	for _, tt := range tests {
+		if got := detectFormat(tt.contentType, tt.url); got != tt.want {
+			t.Errorf("detectFormat(%q, %q) = %q, want %q", tt.contentType, tt.url, got, tt.want)
+		}
+	}
+}
+
+func TestSanitizeURL(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"http://user:pass@example.com/path", "http://example.com/path"},
+		{"https://example.com/path", "https://example.com/path"},
+		{"not a url", "not%20a%20url"},
+		{"http://user@example.com/path", "http://example.com/path"}, // user with no password
+	}
+	for _, tt := range tests {
+		if got := sanitizeURL(tt.input); got != tt.expected {
+			t.Errorf("sanitizeURL(%q) = %q, want %q", tt.input, got, tt.expected)
+		}
+	}
+}
+
+func TestReadConfigFromURL_YAML(t *testing.T) {
+	// Write a YAML config and serve it via httptest.
+	cfg := &config.Config{
+		Services: []*config.ServiceConfig{{Name: "svc", Addr: ":8080"}},
+		Log:      &config.LogConfig{Level: "debug"},
+	}
+	var buf bytes.Buffer
+	if err := cfg.Write(&buf, "yaml"); err != nil {
+		t.Fatal(err)
+	}
+	body := buf.Bytes()
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/yaml")
+		w.Write(body)
+	}))
+	defer ts.Close()
+
+	got := &config.Config{}
+	if err := readConfigFromURL(ts.URL, got); err != nil {
+		t.Fatalf("readConfigFromURL: %v", err)
+	}
+	if len(got.Services) != 1 || got.Services[0].Name != "svc" {
+		t.Fatalf("unexpected services: %+v", got.Services)
+	}
+	if got.Log == nil || got.Log.Level != "debug" {
+		t.Fatalf("unexpected log: %+v", got.Log)
+	}
+}
+
+func TestReadConfigFromURL_JSON(t *testing.T) {
+	// Remote serving JSON with explicit Content-Type.
+	cfg := &config.Config{
+		Services: []*config.ServiceConfig{{Name: "json-svc", Addr: ":9090"}},
+	}
+	var buf bytes.Buffer
+	if err := cfg.Write(&buf, "json"); err != nil {
+		t.Fatal(err)
+	}
+	body := buf.Bytes()
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(body)
+	}))
+	defer ts.Close()
+
+	got := &config.Config{}
+	if err := readConfigFromURL(ts.URL, got); err != nil {
+		t.Fatalf("readConfigFromURL: %v", err)
+	}
+	if got.Services[0].Name != "json-svc" {
+		t.Fatalf("unexpected service name: %s", got.Services[0].Name)
+	}
+}
+
+func TestReadConfigFromURL_Non200(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer ts.Close()
+
+	err := readConfigFromURL(ts.URL, &config.Config{})
+	if err == nil {
+		t.Fatal("expected error for 404 response")
+	}
+}
+
+func TestReadConfigFromURL_OverSize(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/yaml")
+		w.Write(make([]byte, maxConfigSize+1))
+	}))
+	defer ts.Close()
+
+	err := readConfigFromURL(ts.URL, &config.Config{})
+	if err == nil {
+		t.Fatal("expected error for oversized response")
+	}
+}
+
+func TestParse_FileAndURL(t *testing.T) {
+	// Create a local file.
+	fileCfg := &config.Config{
+		Services: []*config.ServiceConfig{{Name: "file-svc", Addr: ":8080"}},
+	}
+	tmpDir := t.TempDir()
+	f, err := os.Create(filepath.Join(tmpDir, "local.yml"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := fileCfg.Write(f, "yaml"); err != nil {
+		f.Close()
+		t.Fatal(err)
+	}
+	f.Close()
+
+	// Serve a second config over HTTP.
+	urlCfg := &config.Config{
+		Services: []*config.ServiceConfig{{Name: "url-svc", Addr: ":8081"}},
+		Log:      &config.LogConfig{Level: "debug"},
+	}
+	var buf bytes.Buffer
+	if err := urlCfg.Write(&buf, "yaml"); err != nil {
+		t.Fatal(err)
+	}
+	body := buf.Bytes()
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/yaml")
+		w.Write(body)
+	}))
+	defer ts.Close()
+
+	Init(Args{
+		CfgFiles: []string{filepath.Join(tmpDir, "local.yml"), ts.URL},
+	})
+
+	got, err := Parse()
+	if err != nil {
+		t.Fatalf("Parse(): %v", err)
+	}
+	if len(got.Services) != 2 {
+		t.Fatalf("expected 2 services, got %d", len(got.Services))
+	}
+	if got.Log == nil || got.Log.Level != "debug" {
+		t.Fatal("Log.Level not set from URL config")
+	}
+}
@@ -2,9 +2,17 @@ package parser

 import (
 	"bufio"
+	"bytes"
 	"encoding/json"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"net/url"
 	"os"
+	"path"
 	"strings"
+	"time"

 	"github.com/go-gost/core/logger"
 	"github.com/go-gost/x/config"
@@ -35,8 +43,9 @@ func Parse() (*config.Config, error) {
 // Args holds the raw CLI flags and positional arguments that Init will merge
 // into the final configuration.
 type Args struct {
-	// CfgFiles is the list of YAML or JSON config file paths, "-" for stdin, or
-	// an inline JSON string. Multiple files are merged left-to-right.
+	// CfgFiles is the list of YAML or JSON config sources: file paths,
+	// HTTP(S) URLs, "-" for stdin, or an inline JSON string.
+	// Multiple sources are merged left-to-right.
 	CfgFiles []string

 	// Services is the list of service definitions from -L flags.
@@ -83,7 +92,8 @@ func (p *parser) Parse() (*config.Config, error) {
 	}
 	cfg = mergeConfig(cfg, cmdCfg)

-	if len(cfg.Services) == 0 && p.args.ApiAddr == "" && cfg.API == nil {
+	if len(cfg.Services) == 0 && p.args.ApiAddr == "" && cfg.API == nil &&
+		len(p.args.CfgFiles) == 0 {
 		if err := cfg.Load(); err != nil {
 			return nil, err
 		}
@@ -179,7 +189,7 @@ func (p *parser) Parse() (*config.Config, error) {
 }

 // readConfig reads a single configuration source, which may be "-" (stdin),
-// an inline JSON string starting with "{", or a file path.
+// an HTTP(S) URL, an inline JSON string starting with "{", or a file path.
 func readConfig(cfgFile string) (*config.Config, error) {
 	cfg := &config.Config{}

@@ -202,6 +212,10 @@ func readConfig(cfgFile string) (*config.Config, error) {
 		if err := json.Unmarshal([]byte(cfgFile), cfg); err != nil {
 			return nil, err
 		}
+	} else if isHTTPURL(cfgFile) { // URL
+		if err := readConfigFromURL(cfgFile, cfg); err != nil {
+			return nil, err
+		}
 	} else {
 		if err := cfg.ReadFile(cfgFile); err != nil { // file
 			return nil, err
@@ -211,6 +225,89 @@ func readConfig(cfgFile string) (*config.Config, error) {
 	return cfg, nil
 }

+// isHTTPURL reports whether s is an HTTP or HTTPS URL.
+func isHTTPURL(s string) bool {
+	u, err := url.Parse(s)
+	return err == nil && u.Scheme != "" && u.Host != "" &&
+		(u.Scheme == "http" || u.Scheme == "https")
+}
+
+// maxConfigSize is the maximum response body size for remote config files.
+const maxConfigSize = 10 << 20 // 10 MiB
+
+// sharedHTTPClient is reused across config URL fetches to benefit from
+// connection pooling.
+var sharedHTTPClient = &http.Client{
+	Timeout: 30 * time.Second,
+}
+
+// readConfigFromURL fetches a configuration from an HTTP(S) URL and reads it
+// into cfg. The format (yaml or json) is detected from the Content-Type header
+// or the URL path extension.
+func readConfigFromURL(urlStr string, cfg *config.Config) error {
+	resp, err := sharedHTTPClient.Get(urlStr)
+	if err != nil {
+		return fmt.Errorf("fetch config from %s: %w", sanitizeURL(urlStr), err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		io.Copy(io.Discard, resp.Body) // drain body for connection reuse; ignore error since we are already failing
+		return fmt.Errorf("fetch config from %s: %s", sanitizeURL(urlStr), resp.Status)
+	}
+
+	data, err := io.ReadAll(io.LimitReader(resp.Body, maxConfigSize+1))
+	if err != nil {
+		return fmt.Errorf("read config from %s: %w", sanitizeURL(urlStr), err)
+	}
+	if len(data) > maxConfigSize {
+		return fmt.Errorf("config from %s exceeds maximum size %d", sanitizeURL(urlStr), maxConfigSize)
+	}
+
+	format := detectFormat(resp.Header.Get("Content-Type"), urlStr)
+
+	if err := cfg.Read(bytes.NewReader(data), format); err != nil {
+		return fmt.Errorf("parse config from %s: %w", sanitizeURL(urlStr), err)
+	}
+	return nil
+}
+
+// sanitizeURL returns the URL with any embedded userinfo stripped for safe
+// use in error messages.
+func sanitizeURL(rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+	u.User = nil
+	return u.String()
+}
+
+// detectFormat determines the config format from the HTTP Content-Type header
+// or the URL path extension. Returns "yaml" or "json".
+func detectFormat(contentType, urlStr string) string {
+	if mediatype, _, err := mime.ParseMediaType(contentType); err == nil {
+		switch mediatype {
+		case "application/json":
+			return "json"
+		case "application/yaml", "application/x-yaml",
+			"text/yaml", "text/x-yaml":
+			return "yaml"
+		}
+	}
+
+	if u, err := url.Parse(urlStr); err == nil {
+		switch strings.ToLower(strings.TrimPrefix(path.Ext(u.Path), ".")) {
+		case "json":
+			return "json"
+		case "yaml", "yml":
+			return "yaml"
+		}
+	}
+
+	return "yaml"
+}
+
 func mergeConfig(cfg1, cfg2 *config.Config) *config.Config {
 	if cfg1 == nil {
 		return cfg2