Browse Source
* WIP playback metrics * Playback metrics collecting + APIs. Closes #793 * Cleanup console messages * Update test * Increase browser test timeout * Update browser tests to not failpull/1786/head
Gabe Kangas
3 years ago
committed by
GitHub
15 changed files with 678 additions and 83 deletions
@ -0,0 +1,60 @@
@@ -0,0 +1,60 @@
|
||||
package admin |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"net/http" |
||||
|
||||
"github.com/owncast/owncast/core" |
||||
"github.com/owncast/owncast/core/data" |
||||
"github.com/owncast/owncast/metrics" |
||||
log "github.com/sirupsen/logrus" |
||||
) |
||||
|
||||
// GetVideoPlaybackMetrics returns video playback metrics.
|
||||
func GetVideoPlaybackMetrics(w http.ResponseWriter, r *http.Request) { |
||||
type response struct { |
||||
Errors []metrics.TimestampedValue `json:"errors"` |
||||
QualityVariantChanges []metrics.TimestampedValue `json:"qualityVariantChanges"` |
||||
Latency []metrics.TimestampedValue `json:"latency"` |
||||
SegmentDownloadDuration []metrics.TimestampedValue `json:"segmentDownloadDuration"` |
||||
SlowestDownloadRate []metrics.TimestampedValue `json:"minPlayerBitrate"` |
||||
AvailableBitrates []int `json:"availableBitrates"` |
||||
SegmentLength int `json:"segmentLength"` |
||||
} |
||||
|
||||
availableBitrates := []int{} |
||||
var segmentLength int |
||||
if core.GetCurrentBroadcast() != nil { |
||||
segmentLength = core.GetCurrentBroadcast().LatencyLevel.SecondsPerSegment |
||||
for _, variants := range core.GetCurrentBroadcast().OutputSettings { |
||||
availableBitrates = append(availableBitrates, variants.VideoBitrate) |
||||
} |
||||
} else { |
||||
segmentLength = data.GetStreamLatencyLevel().SecondsPerSegment |
||||
for _, variants := range data.GetStreamOutputVariants() { |
||||
availableBitrates = append(availableBitrates, variants.VideoBitrate) |
||||
} |
||||
} |
||||
|
||||
errors := metrics.GetPlaybackErrorCountOverTime() |
||||
latency := metrics.GetLatencyOverTime() |
||||
durations := metrics.GetDownloadDurationsOverTime() |
||||
minPlayerBitrate := metrics.GetSlowestDownloadRateOverTime() |
||||
qualityVariantChanges := metrics.GetQualityVariantChangesOverTime() |
||||
|
||||
resp := response{ |
||||
AvailableBitrates: availableBitrates, |
||||
Errors: errors, |
||||
Latency: latency, |
||||
SegmentLength: segmentLength, |
||||
SegmentDownloadDuration: durations, |
||||
SlowestDownloadRate: minPlayerBitrate, |
||||
QualityVariantChanges: qualityVariantChanges, |
||||
} |
||||
|
||||
w.Header().Set("Content-Type", "application/json") |
||||
err := json.NewEncoder(w).Encode(resp) |
||||
if err != nil { |
||||
log.Errorln(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,40 @@
@@ -0,0 +1,40 @@
|
||||
package controllers |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"net/http" |
||||
|
||||
"github.com/owncast/owncast/metrics" |
||||
log "github.com/sirupsen/logrus" |
||||
) |
||||
|
||||
// ReportPlaybackMetrics will accept playback metrics from a client and save
|
||||
// them for future video health reporting.
|
||||
func ReportPlaybackMetrics(w http.ResponseWriter, r *http.Request) { |
||||
if r.Method != POST { |
||||
WriteSimpleResponse(w, false, r.Method+" not supported") |
||||
return |
||||
} |
||||
|
||||
type reportPlaybackMetricsRequest struct { |
||||
Bandwidth float64 `json:"bandwidth"` |
||||
Latency float64 `json:"latency"` |
||||
Errors float64 `json:"errors"` |
||||
DownloadDuration float64 `json:"downloadDuration"` |
||||
QualityVariantChanges float64 `json:"qualityVariantChanges"` |
||||
} |
||||
|
||||
decoder := json.NewDecoder(r.Body) |
||||
var request reportPlaybackMetricsRequest |
||||
if err := decoder.Decode(&request); err != nil { |
||||
log.Errorln("error decoding playback metrics payload:", err) |
||||
WriteSimpleResponse(w, false, err.Error()) |
||||
return |
||||
} |
||||
|
||||
metrics.RegisterPlaybackErrorCount(request.Errors) |
||||
metrics.RegisterPlayerBandwidth(request.Bandwidth) |
||||
metrics.RegisterPlayerLatency(request.Latency) |
||||
metrics.RegisterPlayerSegmentDownloadDuration(request.DownloadDuration) |
||||
metrics.RegisterQualityVariantChangesCount(request.QualityVariantChanges) |
||||
} |
||||
@ -0,0 +1,168 @@
@@ -0,0 +1,168 @@
|
||||
package metrics |
||||
|
||||
import ( |
||||
"math" |
||||
"time" |
||||
|
||||
"github.com/owncast/owncast/utils" |
||||
) |
||||
|
||||
// Playback error counts reported since the last time we collected metrics.
|
||||
var ( |
||||
windowedErrorCounts = []float64{} |
||||
windowedQualityVariantChanges = []float64{} |
||||
windowedBandwidths = []float64{} |
||||
windowedLatencies = []float64{} |
||||
windowedDownloadDurations = []float64{} |
||||
) |
||||
|
||||
// RegisterPlaybackErrorCount will add to the windowed playback error count.
|
||||
func RegisterPlaybackErrorCount(count float64) { |
||||
windowedErrorCounts = append(windowedErrorCounts, count) |
||||
} |
||||
|
||||
// RegisterQualityVariantChangesCount will add to the windowed quality variant
|
||||
// change count.
|
||||
func RegisterQualityVariantChangesCount(count float64) { |
||||
windowedQualityVariantChanges = append(windowedQualityVariantChanges, count) |
||||
} |
||||
|
||||
// RegisterPlayerBandwidth will add to the windowed playback bandwidth.
|
||||
func RegisterPlayerBandwidth(kbps float64) { |
||||
windowedBandwidths = append(windowedBandwidths, kbps) |
||||
} |
||||
|
||||
// RegisterPlayerLatency will add to the windowed player latency values.
|
||||
func RegisterPlayerLatency(seconds float64) { |
||||
windowedLatencies = append(windowedLatencies, seconds) |
||||
} |
||||
|
||||
// RegisterPlayerSegmentDownloadDuration will add to the windowed player segment
|
||||
// download duration values.
|
||||
func RegisterPlayerSegmentDownloadDuration(seconds float64) { |
||||
windowedDownloadDurations = append(windowedDownloadDurations, seconds) |
||||
} |
||||
|
||||
// collectPlaybackErrorCount will take all of the error counts each individual
|
||||
// player reported and average them into a single metric. This is done so
|
||||
// one person with bad connectivity doesn't make it look like everything is
|
||||
// horrible for everyone.
|
||||
func collectPlaybackErrorCount() { |
||||
count := utils.Sum(windowedErrorCounts) |
||||
windowedErrorCounts = []float64{} |
||||
|
||||
metrics.errorCount = append(metrics.errorCount, TimestampedValue{ |
||||
Time: time.Now(), |
||||
Value: count, |
||||
}) |
||||
|
||||
if len(metrics.errorCount) > maxCollectionValues { |
||||
metrics.errorCount = metrics.errorCount[1:] |
||||
} |
||||
|
||||
// Save to Prometheus collector.
|
||||
playbackErrorCount.Set(count) |
||||
} |
||||
|
||||
func collectSegmentDownloadDuration() { |
||||
val := 0.0 |
||||
|
||||
if len(windowedDownloadDurations) > 0 { |
||||
val = utils.Avg(windowedDownloadDurations) |
||||
windowedDownloadDurations = []float64{} |
||||
} |
||||
metrics.segmentDownloadSeconds = append(metrics.segmentDownloadSeconds, TimestampedValue{ |
||||
Time: time.Now(), |
||||
Value: val, |
||||
}) |
||||
|
||||
if len(metrics.segmentDownloadSeconds) > maxCollectionValues { |
||||
metrics.segmentDownloadSeconds = metrics.segmentDownloadSeconds[1:] |
||||
} |
||||
} |
||||
|
||||
// GetDownloadDurationsOverTime will return a window of durations errors over time.
|
||||
func GetDownloadDurationsOverTime() []TimestampedValue { |
||||
return metrics.segmentDownloadSeconds |
||||
} |
||||
|
||||
// GetPlaybackErrorCountOverTime will return a window of playback errors over time.
|
||||
func GetPlaybackErrorCountOverTime() []TimestampedValue { |
||||
return metrics.errorCount |
||||
} |
||||
|
||||
func collectLatencyValues() { |
||||
val := 0.0 |
||||
|
||||
if len(windowedLatencies) > 0 { |
||||
val = utils.Avg(windowedLatencies) |
||||
val = math.Round(val) |
||||
windowedLatencies = []float64{} |
||||
} |
||||
|
||||
metrics.averageLatency = append(metrics.averageLatency, TimestampedValue{ |
||||
Time: time.Now(), |
||||
Value: val, |
||||
}) |
||||
|
||||
if len(metrics.averageLatency) > maxCollectionValues { |
||||
metrics.averageLatency = metrics.averageLatency[1:] |
||||
} |
||||
} |
||||
|
||||
// GetLatencyOverTime will return the min, max and avg latency values over time.
|
||||
func GetLatencyOverTime() []TimestampedValue { |
||||
if len(metrics.averageLatency) == 0 { |
||||
return []TimestampedValue{} |
||||
} |
||||
|
||||
return metrics.averageLatency |
||||
} |
||||
|
||||
// collectLowestBandwidth will collect the lowest bandwidth currently collected
|
||||
// so we can report to the streamer the worst possible streaming condition
|
||||
// being experienced.
|
||||
func collectLowestBandwidth() { |
||||
val := 0.0 |
||||
|
||||
if len(windowedBandwidths) > 0 { |
||||
val, _ = utils.MinMax(windowedBandwidths) |
||||
val = math.Round(val) |
||||
windowedBandwidths = []float64{} |
||||
} |
||||
|
||||
metrics.lowestBitrate = append(metrics.lowestBitrate, TimestampedValue{ |
||||
Time: time.Now(), |
||||
Value: math.Round(val), |
||||
}) |
||||
|
||||
if len(metrics.lowestBitrate) > maxCollectionValues { |
||||
metrics.lowestBitrate = metrics.lowestBitrate[1:] |
||||
} |
||||
} |
||||
|
||||
// GetSlowestDownloadRateOverTime will return the collected lowest bandwidth values
|
||||
// over time.
|
||||
func GetSlowestDownloadRateOverTime() []TimestampedValue { |
||||
if len(metrics.lowestBitrate) == 0 { |
||||
return []TimestampedValue{} |
||||
} |
||||
|
||||
return metrics.lowestBitrate |
||||
} |
||||
|
||||
func collectQualityVariantChanges() { |
||||
count := utils.Sum(windowedQualityVariantChanges) |
||||
windowedQualityVariantChanges = []float64{} |
||||
|
||||
metrics.qualityVariantChanges = append(metrics.qualityVariantChanges, TimestampedValue{ |
||||
Time: time.Now(), |
||||
Value: count, |
||||
}) |
||||
} |
||||
|
||||
// GetQualityVariantChangesOverTime will return the collected quality variant
|
||||
// changes.
|
||||
func GetQualityVariantChangesOverTime() []TimestampedValue { |
||||
return metrics.qualityVariantChanges |
||||
} |
||||
@ -0,0 +1,98 @@
@@ -0,0 +1,98 @@
|
||||
import { URL_PLAYBACK_METRICS } from '../utils/constants.js'; |
||||
const METRICS_SEND_INTERVAL = 10000; |
||||
|
||||
class PlaybackMetrics { |
||||
constructor() { |
||||
this.hasPerformedInitialVariantChange = false; |
||||
|
||||
this.segmentDownloadTime = []; |
||||
this.bandwidthTracking = []; |
||||
this.latencyTracking = []; |
||||
this.errors = 0; |
||||
this.qualityVariantChanges = 0; |
||||
this.isBuffering = false; |
||||
|
||||
setInterval(() => { |
||||
this.send(); |
||||
}, METRICS_SEND_INTERVAL); |
||||
} |
||||
|
||||
incrementErrorCount(count) { |
||||
this.errors += count; |
||||
} |
||||
|
||||
incrementQualityVariantChanges() { |
||||
// We always start the player at the lowest quality, so let's just not
|
||||
// count the first change.
|
||||
if (!this.hasPerformedInitialVariantChange) { |
||||
this.hasPerformedInitialVariantChange = true; |
||||
return; |
||||
} |
||||
this.qualityVariantChanges++; |
||||
} |
||||
|
||||
trackSegmentDownloadTime(seconds) { |
||||
this.segmentDownloadTime.push(seconds); |
||||
} |
||||
|
||||
trackBandwidth(bps) { |
||||
this.bandwidthTracking.push(bps); |
||||
} |
||||
|
||||
trackLatency(latency) { |
||||
this.latencyTracking.push(latency); |
||||
} |
||||
|
||||
async send() { |
||||
if ( |
||||
this.segmentDownloadTime.length < 4 || |
||||
this.bandwidthTracking.length < 4 |
||||
) { |
||||
return; |
||||
} |
||||
|
||||
const errorCount = this.errors; |
||||
const average = (arr) => arr.reduce((p, c) => p + c, 0) / arr.length; |
||||
|
||||
const averageDownloadDuration = average(this.segmentDownloadTime) / 1000; |
||||
const roundedAverageDownloadDuration = |
||||
Math.round(averageDownloadDuration * 1000) / 1000; |
||||
|
||||
const averageBandwidth = average(this.bandwidthTracking) / 1000; |
||||
const roundedAverageBandwidth = Math.round(averageBandwidth * 1000) / 1000; |
||||
|
||||
const averageLatency = average(this.latencyTracking) / 1000; |
||||
const roundedAverageLatency = Math.round(averageLatency * 1000) / 1000; |
||||
|
||||
const data = { |
||||
bandwidth: roundedAverageBandwidth, |
||||
latency: roundedAverageLatency, |
||||
downloadDuration: roundedAverageDownloadDuration, |
||||
errors: errorCount + this.isBuffering ? 1 : 0, |
||||
qualityVariantChanges: this.qualityVariantChanges, |
||||
}; |
||||
this.errors = 0; |
||||
this.qualityVariantChanges = 0; |
||||
this.segmentDownloadTime = []; |
||||
this.bandwidthTracking = []; |
||||
this.latencyTracking = []; |
||||
|
||||
const options = { |
||||
method: 'POST', |
||||
headers: { |
||||
'Content-Type': 'application/json', |
||||
}, |
||||
body: JSON.stringify(data), |
||||
}; |
||||
|
||||
try { |
||||
fetch(URL_PLAYBACK_METRICS, options); |
||||
} catch (e) { |
||||
console.error(e); |
||||
} |
||||
|
||||
// console.log(data);
|
||||
} |
||||
} |
||||
|
||||
export default PlaybackMetrics; |
||||
Loading…
Reference in new issue