Some migration fixes and database optimizations

core/chat/persistence.go

@@ -22,7 +22,7 @@ const (
 
 func setupPersistence() {
 	_datastore = data.GetDatastore()
-	createMessagesTable()
+	data.CreateMessagesTable(_datastore.DB)
 
 	chatDataPruner := time.NewTicker(5 * time.Minute)
 	go func() {
@@ -33,26 +33,6 @@ func setupPersistence() {
 	}()
 }
 
-func createMessagesTable() {
-	createTableSQL := `CREATE TABLE IF NOT EXISTS messages (
-		"id" string NOT NULL PRIMARY KEY,
-		"user_id" INTEGER,
-		"body" TEXT,
-		"eventType" TEXT,
-		"hidden_at" DATETIME,
-		"timestamp" DATETIME
-	);`
-
-	stmt, err := _datastore.DB.Prepare(createTableSQL)
-	if err != nil {
-		log.Fatal("error creating chat messages table", err)
-	}
-	defer stmt.Close()
-	if _, err := stmt.Exec(); err != nil {
-		log.Fatal("error creating chat messages table", err)
-	}
-}
-
 func SaveUserMessage(event events.UserMessageEvent) {
 	saveEvent(event.Id, event.User.Id, event.Body, event.Type, event.HiddenAt, event.Timestamp)
 }
@@ -176,7 +156,7 @@ func GetChatModerationHistory() []events.UserMessageEvent {
 
 func GetChatHistory() []events.UserMessageEvent {
 	// Get all visible messages
-	var query = fmt.Sprintf("SELECT id, user_id, body, eventType, hidden_at, timestamp, display_name, display_color, created_at, disabled_at, previous_names, namechanged_at FROM (SELECT * FROM messages LEFT OUTER JOIN users ON messages.user_id = users.id WHERE hidden_at IS NULL ORDER BY timestamp DESC LIMIT %d) ORDER BY timestamp asc", maxBacklogNumber)
+	var query = fmt.Sprintf("SELECT id, user_id, body, eventType, hidden_at, timestamp, display_name, display_color, created_at, disabled_at, previous_names, namechanged_at FROM (SELECT * FROM messages LEFT OUTER JOIN users ON messages.user_id = users.id WHERE hidden_at IS NULL AND disabled_at IS NULL ORDER BY timestamp DESC LIMIT %d) ORDER BY timestamp asc", maxBacklogNumber)
 	return getChat(query)
 }
 

core/data/data.go

@@ -45,7 +45,7 @@ func SetupPersistence(file string) error {
 		}
 	}
 
-	db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s", file))
+	db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?_cache_size=10000", file))
 	db.SetMaxOpenConns(1)
 	_db = db
 

core/data/messages.go

@@ -0,0 +1,32 @@
+package data
+
+import (
+	"database/sql"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func CreateMessagesTable(db *sql.DB) {
+	createTableSQL := `CREATE TABLE IF NOT EXISTS messages (
+		"id" string NOT NULL,
+		"user_id" INTEGER,
+		"body" TEXT,
+		"eventType" TEXT,
+		"hidden_at" DATETIME,
+		"timestamp" DATETIME,
+		PRIMARY KEY (id)
+	);CREATE INDEX index ON messages (id, user_id, hidden_at, timestamp);
+	CREATE INDEX id ON messages (id);
+	CREATE INDEX user_id ON messages (user_id);
+	CREATE INDEX hidden_at ON messages (hidden_at);
+	CREATE INDEX timestamp ON messages (timestamp);`
+
+	stmt, err := db.Prepare(createTableSQL)
+	if err != nil {
+		log.Fatal("error creating chat messages table", err)
+	}
+	defer stmt.Close()
+	if _, err := stmt.Exec(); err != nil {
+		log.Fatal("error creating chat messages table", err)
+	}
+}

core/data/migrations.go

@@ -25,7 +25,7 @@ func migrateToSchema1(db *sql.DB) {
 	}
 
 	// Recreate it
-	createUsersTable(db)
+	CreateMessagesTable(db)
 
 	// Migrate access tokens to become chat users
 	type oldAccessToken struct {

core/data/users.go

@@ -21,9 +21,11 @@ func createUsersTable(db *sql.DB) {
 		"scopes" TEXT,
 		"type" TEXT DEFAULT 'STANDARD',
 		"last_used" DATETIME DEFAULT CURRENT_TIMESTAMP,
-		PRIMARY KEY (id, access_token),
-		UNIQUE(id, access_token)
-	);CREATE INDEX index ON users (id, access_token)`
+		PRIMARY KEY (id)
+	);CREATE INDEX index ON users (id, access_token, disabled_at);
+	CREATE INDEX id ON users (id);
+	CREATE INDEX access_token ON users (access_token);
+	CREATE INDEX disabled_at ON USERS (disabled_at);`
 
 	stmt, err := db.Prepare(createTableSQL)
 	if err != nil {