Browse Source
* WIP playback metrics * Playback metrics collecting + APIs. Closes #793 * Cleanup console messages * Update test * Increase browser test timeout * Update browser tests to not failpull/1786/head
Gabe Kangas
4 years ago
committed by
GitHub
15 changed files with 678 additions and 83 deletions
@ -0,0 +1,60 @@ |
|||||||
|
package admin |
||||||
|
|
||||||
|
import ( |
||||||
|
"encoding/json" |
||||||
|
"net/http" |
||||||
|
|
||||||
|
"github.com/owncast/owncast/core" |
||||||
|
"github.com/owncast/owncast/core/data" |
||||||
|
"github.com/owncast/owncast/metrics" |
||||||
|
log "github.com/sirupsen/logrus" |
||||||
|
) |
||||||
|
|
||||||
|
// GetVideoPlaybackMetrics returns video playback metrics.
|
||||||
|
func GetVideoPlaybackMetrics(w http.ResponseWriter, r *http.Request) { |
||||||
|
type response struct { |
||||||
|
Errors []metrics.TimestampedValue `json:"errors"` |
||||||
|
QualityVariantChanges []metrics.TimestampedValue `json:"qualityVariantChanges"` |
||||||
|
Latency []metrics.TimestampedValue `json:"latency"` |
||||||
|
SegmentDownloadDuration []metrics.TimestampedValue `json:"segmentDownloadDuration"` |
||||||
|
SlowestDownloadRate []metrics.TimestampedValue `json:"minPlayerBitrate"` |
||||||
|
AvailableBitrates []int `json:"availableBitrates"` |
||||||
|
SegmentLength int `json:"segmentLength"` |
||||||
|
} |
||||||
|
|
||||||
|
availableBitrates := []int{} |
||||||
|
var segmentLength int |
||||||
|
if core.GetCurrentBroadcast() != nil { |
||||||
|
segmentLength = core.GetCurrentBroadcast().LatencyLevel.SecondsPerSegment |
||||||
|
for _, variants := range core.GetCurrentBroadcast().OutputSettings { |
||||||
|
availableBitrates = append(availableBitrates, variants.VideoBitrate) |
||||||
|
} |
||||||
|
} else { |
||||||
|
segmentLength = data.GetStreamLatencyLevel().SecondsPerSegment |
||||||
|
for _, variants := range data.GetStreamOutputVariants() { |
||||||
|
availableBitrates = append(availableBitrates, variants.VideoBitrate) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
errors := metrics.GetPlaybackErrorCountOverTime() |
||||||
|
latency := metrics.GetLatencyOverTime() |
||||||
|
durations := metrics.GetDownloadDurationsOverTime() |
||||||
|
minPlayerBitrate := metrics.GetSlowestDownloadRateOverTime() |
||||||
|
qualityVariantChanges := metrics.GetQualityVariantChangesOverTime() |
||||||
|
|
||||||
|
resp := response{ |
||||||
|
AvailableBitrates: availableBitrates, |
||||||
|
Errors: errors, |
||||||
|
Latency: latency, |
||||||
|
SegmentLength: segmentLength, |
||||||
|
SegmentDownloadDuration: durations, |
||||||
|
SlowestDownloadRate: minPlayerBitrate, |
||||||
|
QualityVariantChanges: qualityVariantChanges, |
||||||
|
} |
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json") |
||||||
|
err := json.NewEncoder(w).Encode(resp) |
||||||
|
if err != nil { |
||||||
|
log.Errorln(err) |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,40 @@ |
|||||||
|
package controllers |
||||||
|
|
||||||
|
import ( |
||||||
|
"encoding/json" |
||||||
|
"net/http" |
||||||
|
|
||||||
|
"github.com/owncast/owncast/metrics" |
||||||
|
log "github.com/sirupsen/logrus" |
||||||
|
) |
||||||
|
|
||||||
|
// ReportPlaybackMetrics will accept playback metrics from a client and save
|
||||||
|
// them for future video health reporting.
|
||||||
|
func ReportPlaybackMetrics(w http.ResponseWriter, r *http.Request) { |
||||||
|
if r.Method != POST { |
||||||
|
WriteSimpleResponse(w, false, r.Method+" not supported") |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
type reportPlaybackMetricsRequest struct { |
||||||
|
Bandwidth float64 `json:"bandwidth"` |
||||||
|
Latency float64 `json:"latency"` |
||||||
|
Errors float64 `json:"errors"` |
||||||
|
DownloadDuration float64 `json:"downloadDuration"` |
||||||
|
QualityVariantChanges float64 `json:"qualityVariantChanges"` |
||||||
|
} |
||||||
|
|
||||||
|
decoder := json.NewDecoder(r.Body) |
||||||
|
var request reportPlaybackMetricsRequest |
||||||
|
if err := decoder.Decode(&request); err != nil { |
||||||
|
log.Errorln("error decoding playback metrics payload:", err) |
||||||
|
WriteSimpleResponse(w, false, err.Error()) |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
metrics.RegisterPlaybackErrorCount(request.Errors) |
||||||
|
metrics.RegisterPlayerBandwidth(request.Bandwidth) |
||||||
|
metrics.RegisterPlayerLatency(request.Latency) |
||||||
|
metrics.RegisterPlayerSegmentDownloadDuration(request.DownloadDuration) |
||||||
|
metrics.RegisterQualityVariantChangesCount(request.QualityVariantChanges) |
||||||
|
} |
||||||
@ -0,0 +1,168 @@ |
|||||||
|
package metrics |
||||||
|
|
||||||
|
import ( |
||||||
|
"math" |
||||||
|
"time" |
||||||
|
|
||||||
|
"github.com/owncast/owncast/utils" |
||||||
|
) |
||||||
|
|
||||||
|
// Playback error counts reported since the last time we collected metrics.
|
||||||
|
var ( |
||||||
|
windowedErrorCounts = []float64{} |
||||||
|
windowedQualityVariantChanges = []float64{} |
||||||
|
windowedBandwidths = []float64{} |
||||||
|
windowedLatencies = []float64{} |
||||||
|
windowedDownloadDurations = []float64{} |
||||||
|
) |
||||||
|
|
||||||
|
// RegisterPlaybackErrorCount will add to the windowed playback error count.
|
||||||
|
func RegisterPlaybackErrorCount(count float64) { |
||||||
|
windowedErrorCounts = append(windowedErrorCounts, count) |
||||||
|
} |
||||||
|
|
||||||
|
// RegisterQualityVariantChangesCount will add to the windowed quality variant
|
||||||
|
// change count.
|
||||||
|
func RegisterQualityVariantChangesCount(count float64) { |
||||||
|
windowedQualityVariantChanges = append(windowedQualityVariantChanges, count) |
||||||
|
} |
||||||
|
|
||||||
|
// RegisterPlayerBandwidth will add to the windowed playback bandwidth.
|
||||||
|
func RegisterPlayerBandwidth(kbps float64) { |
||||||
|
windowedBandwidths = append(windowedBandwidths, kbps) |
||||||
|
} |
||||||
|
|
||||||
|
// RegisterPlayerLatency will add to the windowed player latency values.
|
||||||
|
func RegisterPlayerLatency(seconds float64) { |
||||||
|
windowedLatencies = append(windowedLatencies, seconds) |
||||||
|
} |
||||||
|
|
||||||
|
// RegisterPlayerSegmentDownloadDuration will add to the windowed player segment
|
||||||
|
// download duration values.
|
||||||
|
func RegisterPlayerSegmentDownloadDuration(seconds float64) { |
||||||
|
windowedDownloadDurations = append(windowedDownloadDurations, seconds) |
||||||
|
} |
||||||
|
|
||||||
|
// collectPlaybackErrorCount will take all of the error counts each individual
|
||||||
|
// player reported and average them into a single metric. This is done so
|
||||||
|
// one person with bad connectivity doesn't make it look like everything is
|
||||||
|
// horrible for everyone.
|
||||||
|
func collectPlaybackErrorCount() { |
||||||
|
count := utils.Sum(windowedErrorCounts) |
||||||
|
windowedErrorCounts = []float64{} |
||||||
|
|
||||||
|
metrics.errorCount = append(metrics.errorCount, TimestampedValue{ |
||||||
|
Time: time.Now(), |
||||||
|
Value: count, |
||||||
|
}) |
||||||
|
|
||||||
|
if len(metrics.errorCount) > maxCollectionValues { |
||||||
|
metrics.errorCount = metrics.errorCount[1:] |
||||||
|
} |
||||||
|
|
||||||
|
// Save to Prometheus collector.
|
||||||
|
playbackErrorCount.Set(count) |
||||||
|
} |
||||||
|
|
||||||
|
func collectSegmentDownloadDuration() { |
||||||
|
val := 0.0 |
||||||
|
|
||||||
|
if len(windowedDownloadDurations) > 0 { |
||||||
|
val = utils.Avg(windowedDownloadDurations) |
||||||
|
windowedDownloadDurations = []float64{} |
||||||
|
} |
||||||
|
metrics.segmentDownloadSeconds = append(metrics.segmentDownloadSeconds, TimestampedValue{ |
||||||
|
Time: time.Now(), |
||||||
|
Value: val, |
||||||
|
}) |
||||||
|
|
||||||
|
if len(metrics.segmentDownloadSeconds) > maxCollectionValues { |
||||||
|
metrics.segmentDownloadSeconds = metrics.segmentDownloadSeconds[1:] |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// GetDownloadDurationsOverTime will return a window of durations errors over time.
|
||||||
|
func GetDownloadDurationsOverTime() []TimestampedValue { |
||||||
|
return metrics.segmentDownloadSeconds |
||||||
|
} |
||||||
|
|
||||||
|
// GetPlaybackErrorCountOverTime will return a window of playback errors over time.
|
||||||
|
func GetPlaybackErrorCountOverTime() []TimestampedValue { |
||||||
|
return metrics.errorCount |
||||||
|
} |
||||||
|
|
||||||
|
func collectLatencyValues() { |
||||||
|
val := 0.0 |
||||||
|
|
||||||
|
if len(windowedLatencies) > 0 { |
||||||
|
val = utils.Avg(windowedLatencies) |
||||||
|
val = math.Round(val) |
||||||
|
windowedLatencies = []float64{} |
||||||
|
} |
||||||
|
|
||||||
|
metrics.averageLatency = append(metrics.averageLatency, TimestampedValue{ |
||||||
|
Time: time.Now(), |
||||||
|
Value: val, |
||||||
|
}) |
||||||
|
|
||||||
|
if len(metrics.averageLatency) > maxCollectionValues { |
||||||
|
metrics.averageLatency = metrics.averageLatency[1:] |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// GetLatencyOverTime will return the min, max and avg latency values over time.
|
||||||
|
func GetLatencyOverTime() []TimestampedValue { |
||||||
|
if len(metrics.averageLatency) == 0 { |
||||||
|
return []TimestampedValue{} |
||||||
|
} |
||||||
|
|
||||||
|
return metrics.averageLatency |
||||||
|
} |
||||||
|
|
||||||
|
// collectLowestBandwidth will collect the lowest bandwidth currently collected
|
||||||
|
// so we can report to the streamer the worst possible streaming condition
|
||||||
|
// being experienced.
|
||||||
|
func collectLowestBandwidth() { |
||||||
|
val := 0.0 |
||||||
|
|
||||||
|
if len(windowedBandwidths) > 0 { |
||||||
|
val, _ = utils.MinMax(windowedBandwidths) |
||||||
|
val = math.Round(val) |
||||||
|
windowedBandwidths = []float64{} |
||||||
|
} |
||||||
|
|
||||||
|
metrics.lowestBitrate = append(metrics.lowestBitrate, TimestampedValue{ |
||||||
|
Time: time.Now(), |
||||||
|
Value: math.Round(val), |
||||||
|
}) |
||||||
|
|
||||||
|
if len(metrics.lowestBitrate) > maxCollectionValues { |
||||||
|
metrics.lowestBitrate = metrics.lowestBitrate[1:] |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// GetSlowestDownloadRateOverTime will return the collected lowest bandwidth values
|
||||||
|
// over time.
|
||||||
|
func GetSlowestDownloadRateOverTime() []TimestampedValue { |
||||||
|
if len(metrics.lowestBitrate) == 0 { |
||||||
|
return []TimestampedValue{} |
||||||
|
} |
||||||
|
|
||||||
|
return metrics.lowestBitrate |
||||||
|
} |
||||||
|
|
||||||
|
func collectQualityVariantChanges() { |
||||||
|
count := utils.Sum(windowedQualityVariantChanges) |
||||||
|
windowedQualityVariantChanges = []float64{} |
||||||
|
|
||||||
|
metrics.qualityVariantChanges = append(metrics.qualityVariantChanges, TimestampedValue{ |
||||||
|
Time: time.Now(), |
||||||
|
Value: count, |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
// GetQualityVariantChangesOverTime will return the collected quality variant
|
||||||
|
// changes.
|
||||||
|
func GetQualityVariantChangesOverTime() []TimestampedValue { |
||||||
|
return metrics.qualityVariantChanges |
||||||
|
} |
||||||
@ -0,0 +1,98 @@ |
|||||||
|
import { URL_PLAYBACK_METRICS } from '../utils/constants.js'; |
||||||
|
const METRICS_SEND_INTERVAL = 10000; |
||||||
|
|
||||||
|
class PlaybackMetrics { |
||||||
|
constructor() { |
||||||
|
this.hasPerformedInitialVariantChange = false; |
||||||
|
|
||||||
|
this.segmentDownloadTime = []; |
||||||
|
this.bandwidthTracking = []; |
||||||
|
this.latencyTracking = []; |
||||||
|
this.errors = 0; |
||||||
|
this.qualityVariantChanges = 0; |
||||||
|
this.isBuffering = false; |
||||||
|
|
||||||
|
setInterval(() => { |
||||||
|
this.send(); |
||||||
|
}, METRICS_SEND_INTERVAL); |
||||||
|
} |
||||||
|
|
||||||
|
incrementErrorCount(count) { |
||||||
|
this.errors += count; |
||||||
|
} |
||||||
|
|
||||||
|
incrementQualityVariantChanges() { |
||||||
|
// We always start the player at the lowest quality, so let's just not
|
||||||
|
// count the first change.
|
||||||
|
if (!this.hasPerformedInitialVariantChange) { |
||||||
|
this.hasPerformedInitialVariantChange = true; |
||||||
|
return; |
||||||
|
} |
||||||
|
this.qualityVariantChanges++; |
||||||
|
} |
||||||
|
|
||||||
|
trackSegmentDownloadTime(seconds) { |
||||||
|
this.segmentDownloadTime.push(seconds); |
||||||
|
} |
||||||
|
|
||||||
|
trackBandwidth(bps) { |
||||||
|
this.bandwidthTracking.push(bps); |
||||||
|
} |
||||||
|
|
||||||
|
trackLatency(latency) { |
||||||
|
this.latencyTracking.push(latency); |
||||||
|
} |
||||||
|
|
||||||
|
async send() { |
||||||
|
if ( |
||||||
|
this.segmentDownloadTime.length < 4 || |
||||||
|
this.bandwidthTracking.length < 4 |
||||||
|
) { |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
const errorCount = this.errors; |
||||||
|
const average = (arr) => arr.reduce((p, c) => p + c, 0) / arr.length; |
||||||
|
|
||||||
|
const averageDownloadDuration = average(this.segmentDownloadTime) / 1000; |
||||||
|
const roundedAverageDownloadDuration = |
||||||
|
Math.round(averageDownloadDuration * 1000) / 1000; |
||||||
|
|
||||||
|
const averageBandwidth = average(this.bandwidthTracking) / 1000; |
||||||
|
const roundedAverageBandwidth = Math.round(averageBandwidth * 1000) / 1000; |
||||||
|
|
||||||
|
const averageLatency = average(this.latencyTracking) / 1000; |
||||||
|
const roundedAverageLatency = Math.round(averageLatency * 1000) / 1000; |
||||||
|
|
||||||
|
const data = { |
||||||
|
bandwidth: roundedAverageBandwidth, |
||||||
|
latency: roundedAverageLatency, |
||||||
|
downloadDuration: roundedAverageDownloadDuration, |
||||||
|
errors: errorCount + this.isBuffering ? 1 : 0, |
||||||
|
qualityVariantChanges: this.qualityVariantChanges, |
||||||
|
}; |
||||||
|
this.errors = 0; |
||||||
|
this.qualityVariantChanges = 0; |
||||||
|
this.segmentDownloadTime = []; |
||||||
|
this.bandwidthTracking = []; |
||||||
|
this.latencyTracking = []; |
||||||
|
|
||||||
|
const options = { |
||||||
|
method: 'POST', |
||||||
|
headers: { |
||||||
|
'Content-Type': 'application/json', |
||||||
|
}, |
||||||
|
body: JSON.stringify(data), |
||||||
|
}; |
||||||
|
|
||||||
|
try { |
||||||
|
fetch(URL_PLAYBACK_METRICS, options); |
||||||
|
} catch (e) { |
||||||
|
console.error(e); |
||||||
|
} |
||||||
|
|
||||||
|
// console.log(data);
|
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
export default PlaybackMetrics; |
||||||
Loading…
Reference in new issue