add new metrics for chain node

chain/route.go

@@ -100,6 +100,8 @@ func (r *Route) connect(ctx context.Context) (conn net.Conn, err error) {
 		node.Marker.Mark()
 		return
 	}
+
+	start := time.Now()
 	cc, err := node.Transport.Dial(ctx, addr)
 	if err != nil {
 		node.Marker.Mark()
@@ -114,6 +116,9 @@ func (r *Route) connect(ctx context.Context) (conn net.Conn, err error) {
 	}
 	node.Marker.Reset()
 
+	metrics.ChainNodeConnectSeconds(r.chain.name, node.Name).
+		Observe(time.Since(start).Seconds())
+
 	preNode := node
 	for _, node := range r.nodes[1:] {
 		addr, err = resolve(ctx, network, node.Addr, node.Resolver, node.Hosts, r.logger)

metrics/metrics.go

@@ -36,6 +36,7 @@ type Metrics struct {
 	requests                 *prometheus.CounterVec
 	requestsInFlight         *prometheus.GaugeVec
 	requestSeconds           *prometheus.HistogramVec
+	chainNodeConnectSecconds *prometheus.HistogramVec
 	inputBytes               *prometheus.CounterVec
 	outputBytes              *prometheus.CounterVec
 	handlerErrors            *prometheus.CounterVec
@@ -76,6 +77,15 @@ func NewMetrics() *Metrics {
 				},
 			},
 			[]string{"host", "service"}),
+		chainNodeConnectSecconds: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Name: "gost_chain_node_connect_duration_seconds",
+				Help: "Distribution of chain node connect latencies",
+				Buckets: []float64{
+					.01, .05, .1, .25, .5, 1, 1.5, 2, 5, 10, 15, 30, 60,
+				},
+			},
+			[]string{"host", "chain", "node"}),
 		inputBytes: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
 				Name: "gost_service_transfer_input_bytes_total",
@@ -105,6 +115,7 @@ func NewMetrics() *Metrics {
 	prometheus.MustRegister(m.requests)
 	prometheus.MustRegister(m.requestsInFlight)
 	prometheus.MustRegister(m.requestSeconds)
+	prometheus.MustRegister(m.chainNodeConnectSecconds)
 	prometheus.MustRegister(m.inputBytes)
 	prometheus.MustRegister(m.outputBytes)
 	prometheus.MustRegister(m.handlerErrors)
@@ -156,6 +167,18 @@ func RequestSeconds(service string) Observer {
 		})
 }
 
+func ChainNodeConnectSeconds(chain, node string) Observer {
+	if metrics == nil || metrics.chainNodeConnectSecconds == nil {
+		return nilObserver
+	}
+	return metrics.chainNodeConnectSecconds.
+		With(prometheus.Labels{
+			"host":  metrics.host,
+			"chain": chain,
+			"node":  node,
+		})
+}
+
 func InputBytes(service string) Counter {
 	if metrics == nil || metrics.inputBytes == nil {
 		return nilCounter