diff --git a/controllers/admin/hardware.go b/controllers/admin/hardware.go new file mode 100644 index 000000000..267c48bc4 --- /dev/null +++ b/controllers/admin/hardware.go @@ -0,0 +1,16 @@ +package admin + +import ( + "encoding/json" + "net/http" + + "github.com/gabek/owncast/metrics" +) + +// GetHardwareStats will return hardware utilization over time +func GetHardwareStats(w http.ResponseWriter, r *http.Request) { + metrics := metrics.Metrics + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(metrics) +} diff --git a/metrics/alerting.go b/metrics/alerting.go index b8e7107d0..08994cbeb 100644 --- a/metrics/alerting.go +++ b/metrics/alerting.go @@ -6,12 +6,14 @@ import ( const maxCPUAlertingThresholdPCT = 95 const maxRAMAlertingThresholdPCT = 95 +const maxDiskAlertingThresholdPCT = 95 -const alertingError = "The %s utilization of %d%% is higher than the alerting threshold of %d%%. This can cause issues with video generation and delivery. Please visit the documentation at http://owncast.online/docs/troubleshooting/ to help troubleshoot this issue." +const alertingError = "The %s utilization of %d%% can cause issues with video generation and delivery. Please visit the documentation at http://owncast.online/docs/troubleshooting/ to help troubleshoot this issue." func handleAlerting() { handleCPUAlerting() handleRAMAlerting() + handleDiskAlerting() } func handleCPUAlerting() { @@ -21,7 +23,7 @@ func handleCPUAlerting() { avg := recentAverage(Metrics.CPUUtilizations) if avg > maxCPUAlertingThresholdPCT { - log.Errorf(alertingError, "CPU", avg, maxCPUAlertingThresholdPCT) + log.Errorf(alertingError, "CPU", maxCPUAlertingThresholdPCT) } } @@ -32,7 +34,19 @@ func handleRAMAlerting() { avg := recentAverage(Metrics.RAMUtilizations) if avg > maxRAMAlertingThresholdPCT { - log.Errorf(alertingError, "memory", avg, maxRAMAlertingThresholdPCT) + log.Errorf(alertingError, "memory", maxRAMAlertingThresholdPCT) + } +} + +func handleDiskAlerting() { + if len(Metrics.DiskUtilizations) < 2 { + return + } + + avg := recentAverage(Metrics.DiskUtilizations) + + if avg > maxDiskAlertingThresholdPCT { + log.Errorf(alertingError, "disk", maxRAMAlertingThresholdPCT) } } diff --git a/metrics/hardware.go b/metrics/hardware.go index 4c0687c21..92ee9a175 100644 --- a/metrics/hardware.go +++ b/metrics/hardware.go @@ -4,6 +4,7 @@ import ( "time" "github.com/shirou/gopsutil/cpu" + "github.com/shirou/gopsutil/disk" "github.com/shirou/gopsutil/mem" ) @@ -33,3 +34,15 @@ func collectRAMUtilization() { metricValue := timestampedValue{time.Now(), int(memoryUsage.UsedPercent)} Metrics.RAMUtilizations = append(Metrics.RAMUtilizations, metricValue) } + +func collectDiskUtilization() { + path := "./" + diskUse, _ := disk.Usage(path) + + if len(Metrics.DiskUtilizations) > maxCollectionValues { + Metrics.DiskUtilizations = Metrics.DiskUtilizations[1:] + } + + metricValue := timestampedValue{time.Now(), int(diskUse.UsedPercent)} + Metrics.DiskUtilizations = append(Metrics.DiskUtilizations, metricValue) +} diff --git a/metrics/metrics.go b/metrics/metrics.go index 4c00594b7..a32345a6d 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -5,21 +5,25 @@ import ( ) // How often we poll for updates -const metricsPollingInterval = 15 * time.Second +const metricsPollingInterval = 1 * time.Minute -type metrics struct { - CPUUtilizations []timestampedValue - RAMUtilizations []timestampedValue - Viewers []timestampedValue +// CollectedMetrics stores different collected + timestamped values +type CollectedMetrics struct { + CPUUtilizations []timestampedValue `json:"cpu"` + RAMUtilizations []timestampedValue `json:"memory"` + DiskUtilizations []timestampedValue `json:"disk"` + + Viewers []timestampedValue `json:"-"` } // Metrics is the shared Metrics instance -var Metrics *metrics +var Metrics *CollectedMetrics // Start will begin the metrics collection and alerting func Start() { - Metrics = new(metrics) - startViewerCollectionMetrics() + Metrics = new(CollectedMetrics) + go startViewerCollectionMetrics() + handlePolling() for range time.Tick(metricsPollingInterval) { handlePolling() @@ -30,6 +34,7 @@ func handlePolling() { // Collect hardware stats collectCPUUtilization() collectRAMUtilization() + collectDiskUtilization() // Alerting handleAlerting() diff --git a/router/router.go b/router/router.go index 6f42f5928..4acae1821 100644 --- a/router/router.go +++ b/router/router.go @@ -66,6 +66,9 @@ func Start() error { // Get viewer count over time http.HandleFunc("/api/admin/viewersOverTime", middleware.RequireAdminAuth(admin.GetViewersOverTime)) + // Get hardware stats + http.HandleFunc("/api/admin/hardwarestats", middleware.RequireAdminAuth(admin.GetHardwareStats)) + port := config.Config.GetPublicWebServerPort() log.Infof("Web server running on port: %d", port)