feat(metrics): add metrics from cloud-provider library (#824)

This includes metrics about internal operations from
`k8s.io/cloud-provider` like the workqueue depth and requests to the
Kubernetes API.

This metrics were already exposed on `:8233/metrics` but this was not
documented or scraped.

This commit now uses the same registry for our metrics and the
Kubernetes libraries, and also exposes them on both ports for backwards
compatibility.

Besides having all data available, this will also help us with debugging
#661.

Co-authored-by: Lukas Metzner <lukas.metzner@hetzner-cloud.de>

Author: apricote

internal/metrics/metrics.go

@@ -21,7 +21,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"k8s.io/component-base/metrics/legacyregistry"
 	"k8s.io/klog/v2"
 )
 
@@ -38,24 +38,20 @@ var (
 	}, []string{"op"})
 )
 
-var registry = prometheus.NewRegistry()
+func init() {
+	GetRegistry().MustRegister(OperationCalled)
+}
 
-func GetRegistry() *prometheus.Registry {
-	return registry
+func GetRegistry() prometheus.Registerer {
+	return legacyregistry.Registerer()
 }
 
 func GetHandler() http.Handler {
-	registry.MustRegister(OperationCalled)
-
-	gatherers := prometheus.Gatherers{
-		prometheus.DefaultGatherer,
-		registry,
-	}
-
-	return promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{})
+	return legacyregistry.Handler()
 }
 
 func Serve(address string) {
+	// The metrics are also served by k8s.io/cloud-provider on the secure serving port.
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", GetHandler())