zhebrak / nvidia_smi_exporter

nvidia-smi exporter for Prometheus

nvidia_smi_exporter

nvidia-smi metrics exporter for Prometheus

Build

> go build -v nvidia_smi_exporter

Run

> ./nvidia_smi_exporter [<port>]

Default port is 9101

localhost:9101/metrics

temperature_gpu{gpu="TITAN X (Pascal)[0]"} 41
utilization_gpu{gpu="TITAN X (Pascal)[0]"} 0
utilization_memory{gpu="TITAN X (Pascal)[0]"} 0
memory_total{gpu="TITAN X (Pascal)[0]"} 12189
memory_free{gpu="TITAN X (Pascal)[0]"} 12189
memory_used{gpu="TITAN X (Pascal)[0]"} 0
temperature_gpu{gpu="TITAN X (Pascal)[1]"} 78
utilization_gpu{gpu="TITAN X (Pascal)[1]"} 95
utilization_memory{gpu="TITAN X (Pascal)[1]"} 59
memory_total{gpu="TITAN X (Pascal)[1]"} 12189
memory_free{gpu="TITAN X (Pascal)[1]"} 1738
memory_used{gpu="TITAN X (Pascal)[1]"} 10451
temperature_gpu{gpu="TITAN X (Pascal)[2]"} 83
utilization_gpu{gpu="TITAN X (Pascal)[2]"} 99
utilization_memory{gpu="TITAN X (Pascal)[2]"} 82
memory_total{gpu="TITAN X (Pascal)[2]"} 12189
memory_free{gpu="TITAN X (Pascal)[2]"} 190
memory_used{gpu="TITAN X (Pascal)[2]"} 11999
temperature_gpu{gpu="TITAN X (Pascal)[3]"} 84
utilization_gpu{gpu="TITAN X (Pascal)[3]"} 97
utilization_memory{gpu="TITAN X (Pascal)[3]"} 76
memory_total{gpu="TITAN X (Pascal)[3]"} 12189
memory_free{gpu="TITAN X (Pascal)[3]"} 536
memory_used{gpu="TITAN X (Pascal)[3]"} 11653

Exact command

nvidia-smi --query-gpu=name,index,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv,noheader,nounits

Prometheus example config

- job_name: "gpu_exporter"
  static_configs:
  - targets: ['localhost:9101']

nvidia_smi_exporter's Issues

Add decimal point support

--- nvidia_smi_exporter.go	2018-03-28 18:33:36.000000000 +0900
+++ edit-nvidia_smi_exporter.go	2018-03-28 17:56:25.000000000 +0900
@@ -13,12 +13,12 @@

 func metrics(response http.ResponseWriter, request *http.Request) {
     out, err := exec.Command(
         "nvidia-smi",
-        "--query-gpu=name,index,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used",
+        "--query-gpu=name,index,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used,power.draw",
         "--format=csv,noheader,nounits").Output()

     if err != nil {
@@ -37,19 +37,22 @@

     metricList := []string {
         "temperature.gpu", "utilization.gpu",
-        "utilization.memory", "memory.total", "memory.free", "memory.used"}
+        "utilization.memory", "memory.total", "memory.free", "memory.used", "power.draw" }

     result := ""
     for _, row := range records {
         name := fmt.Sprintf("%s[%s]", row[0], row[1])
         for idx, value := range row[2:] {
+            mydata := strings.Replace(metricList[idx], ".", "_", -1)
             result = fmt.Sprintf(
                 "%s%s{gpu=\"%s\"} %s\n", result,
-                metricList[idx], name, value)
+                //metricList[idx], name, value)
+                mydata, name, value)
         }
     }

-    fmt.Fprintf(response, strings.Replace(result, ".", "_", -1))
+    //fmt.Fprintf(response, strings.Replace(result, ".", "_", -1))
+    fmt.Fprintf(response, result)
 }

 func main() {