feat(parser): support HTTP(S) URL config sources with -C flag

Add remote config fetching capability to the -C CLI flag, enabling
centralized fleet management of gost nodes.

- readConfig() now detects http:// and https:// URLs and fetches
  them via HTTP GET with a 30s timeout and shared client pool
- Format auto-detection from Content-Type header (using mime
  parsing) with fallback to URL path extension (.yaml/.json)
- 10 MiB response size limit to prevent memory exhaustion
- URL userinfo stripped from error messages for safety
- Fix ReadFile to force viper config type from file extension,
  preventing format detection ambiguity
- Fix viper config search path fallback: skip cfg.Load() when
  -C flags are explicitly provided, avoiding unwanted merges
- Tests: URL fetch (YAML/JSON), non-200, oversize, detectFormat,
  isHTTPURL, sanitizeURL, and combined file+URL Parse scenario
This commit is contained in:
ginuerzh
2026-06-20 20:11:43 +08:00
parent 9c585e2de8
commit e583f5b538
3 changed files with 297 additions and 4 deletions
+101 -4
View File
@@ -2,9 +2,17 @@ package parser
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"mime"
"net/http"
"net/url"
"os"
"path"
"strings"
"time"
"github.com/go-gost/core/logger"
"github.com/go-gost/x/config"
@@ -35,8 +43,9 @@ func Parse() (*config.Config, error) {
// Args holds the raw CLI flags and positional arguments that Init will merge
// into the final configuration.
type Args struct {
// CfgFiles is the list of YAML or JSON config file paths, "-" for stdin, or
// an inline JSON string. Multiple files are merged left-to-right.
// CfgFiles is the list of YAML or JSON config sources: file paths,
// HTTP(S) URLs, "-" for stdin, or an inline JSON string.
// Multiple sources are merged left-to-right.
CfgFiles []string
// Services is the list of service definitions from -L flags.
@@ -83,7 +92,8 @@ func (p *parser) Parse() (*config.Config, error) {
}
cfg = mergeConfig(cfg, cmdCfg)
if len(cfg.Services) == 0 && p.args.ApiAddr == "" && cfg.API == nil {
if len(cfg.Services) == 0 && p.args.ApiAddr == "" && cfg.API == nil &&
len(p.args.CfgFiles) == 0 {
if err := cfg.Load(); err != nil {
return nil, err
}
@@ -179,7 +189,7 @@ func (p *parser) Parse() (*config.Config, error) {
}
// readConfig reads a single configuration source, which may be "-" (stdin),
// an inline JSON string starting with "{", or a file path.
// an HTTP(S) URL, an inline JSON string starting with "{", or a file path.
func readConfig(cfgFile string) (*config.Config, error) {
cfg := &config.Config{}
@@ -202,6 +212,10 @@ func readConfig(cfgFile string) (*config.Config, error) {
if err := json.Unmarshal([]byte(cfgFile), cfg); err != nil {
return nil, err
}
} else if isHTTPURL(cfgFile) { // URL
if err := readConfigFromURL(cfgFile, cfg); err != nil {
return nil, err
}
} else {
if err := cfg.ReadFile(cfgFile); err != nil { // file
return nil, err
@@ -211,6 +225,89 @@ func readConfig(cfgFile string) (*config.Config, error) {
return cfg, nil
}
// isHTTPURL reports whether s is an HTTP or HTTPS URL.
func isHTTPURL(s string) bool {
u, err := url.Parse(s)
return err == nil && u.Scheme != "" && u.Host != "" &&
(u.Scheme == "http" || u.Scheme == "https")
}
// maxConfigSize is the maximum response body size for remote config files.
const maxConfigSize = 10 << 20 // 10 MiB
// sharedHTTPClient is reused across config URL fetches to benefit from
// connection pooling.
var sharedHTTPClient = &http.Client{
Timeout: 30 * time.Second,
}
// readConfigFromURL fetches a configuration from an HTTP(S) URL and reads it
// into cfg. The format (yaml or json) is detected from the Content-Type header
// or the URL path extension.
func readConfigFromURL(urlStr string, cfg *config.Config) error {
resp, err := sharedHTTPClient.Get(urlStr)
if err != nil {
return fmt.Errorf("fetch config from %s: %w", sanitizeURL(urlStr), err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
io.Copy(io.Discard, resp.Body) // drain body for connection reuse; ignore error since we are already failing
return fmt.Errorf("fetch config from %s: %s", sanitizeURL(urlStr), resp.Status)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, maxConfigSize+1))
if err != nil {
return fmt.Errorf("read config from %s: %w", sanitizeURL(urlStr), err)
}
if len(data) > maxConfigSize {
return fmt.Errorf("config from %s exceeds maximum size %d", sanitizeURL(urlStr), maxConfigSize)
}
format := detectFormat(resp.Header.Get("Content-Type"), urlStr)
if err := cfg.Read(bytes.NewReader(data), format); err != nil {
return fmt.Errorf("parse config from %s: %w", sanitizeURL(urlStr), err)
}
return nil
}
// sanitizeURL returns the URL with any embedded userinfo stripped for safe
// use in error messages.
func sanitizeURL(rawURL string) string {
u, err := url.Parse(rawURL)
if err != nil {
return rawURL
}
u.User = nil
return u.String()
}
// detectFormat determines the config format from the HTTP Content-Type header
// or the URL path extension. Returns "yaml" or "json".
func detectFormat(contentType, urlStr string) string {
if mediatype, _, err := mime.ParseMediaType(contentType); err == nil {
switch mediatype {
case "application/json":
return "json"
case "application/yaml", "application/x-yaml",
"text/yaml", "text/x-yaml":
return "yaml"
}
}
if u, err := url.Parse(urlStr); err == nil {
switch strings.ToLower(strings.TrimPrefix(path.Ext(u.Path), ".")) {
case "json":
return "json"
case "yaml", "yml":
return "yaml"
}
}
return "yaml"
}
func mergeConfig(cfg1, cfg2 *config.Config) *config.Config {
if cfg1 == nil {
return cfg2