Add database migrations to support software titles feature (#15401)

Issue #15222

go.mod

@@ -249,6 +249,7 @@ require (
 	github.com/kevinburke/ssh_config v1.1.0 // indirect
 	github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d // indirect
 	github.com/klauspost/compress v1.16.5 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -298,6 +299,7 @@ require (
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	github.com/yashtewari/glob-intersection v0.1.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	github.com/ziutek/mymysql v1.5.4 // indirect
 	go.elastic.co/apm/module/apmhttp/v2 v2.3.0 // indirect
 	go.elastic.co/fastjson v1.1.0 // indirect
 	go.opencensus.io v0.24.0 // indirect

go.sum

@@ -1226,6 +1226,7 @@ github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPR
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
+github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
 github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
 go.elastic.co/apm/module/apmgorilla/v2 v2.3.0 h1:jHw8N252UTwKTk945+Am8AaawhHC6DWpFVeTXQO8Gko=
 go.elastic.co/apm/module/apmgorilla/v2 v2.3.0/go.mod h1:2LXDBbVhFf9rF65jZecvl78IZMuvSRldQ+9A/fjfIo0=

server/datastore/mysql/migrations/tables/20231130132828_AddTableSoftwareTitles.go

@@ -0,0 +1,29 @@
+package tables
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20231130132828, Down_20231130132828)
+}
+
+func Up_20231130132828(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+CREATE TABLE software_titles (
+	id INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
+	name VARCHAR(255) NOT NULL,
+	source VARCHAR(64) NOT NULL,
+	PRIMARY KEY (id),
+	UNIQUE KEY idx_software_titles_name_source (name, source) 
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;`)
+	if err != nil {
+		return fmt.Errorf("failed to create software_titles table: %w", err)
+	}
+	return nil
+}
+
+func Down_20231130132828(tx *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/migrations/tables/20231130132828_AddTableSoftwareTitles_test.go

@@ -0,0 +1,46 @@
+package tables
+
+import (
+	"context"
+	"testing"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUp_20231130132828(t *testing.T) {
+	db := applyUpToPrev(t)
+
+	applyNext(t, db)
+
+	insertStmt := "INSERT INTO software_titles (name, source) VALUES (?, ?)"
+
+	_, err := db.Exec(insertStmt, "test-name", "test-source")
+	require.NoError(t, err)
+
+	// unique constraint applies to name+source
+	_, err = db.Exec(insertStmt, "test-name", "test-source")
+	require.ErrorContains(t, err, "Duplicate entry")
+
+	_, err = db.Exec(insertStmt, "test-name", "test-source2")
+	require.NoError(t, err)
+
+	_, err = db.Exec(insertStmt, "test-name2", "test-source")
+	require.NoError(t, err)
+
+	_, err = db.Exec(insertStmt, "test-name2", "test-source2")
+	require.NoError(t, err)
+
+	_, err = db.Exec(insertStmt, "test-name", "test-name")
+	require.NoError(t, err)
+
+	selectStmt := "SELECT id, name, source FROM software_titles"
+	var rows []struct {
+		ID     uint   `db:"id"`
+		Name   string `db:"name"`
+		Source string `db:"source"`
+	}
+	err = sqlx.SelectContext(context.Background(), db, &rows, selectStmt)
+	require.NoError(t, err)
+	require.Len(t, rows, 5)
+}

server/datastore/mysql/migrations/tables/20231130132931_AlterSoftwareAddTitleId.go

@@ -0,0 +1,28 @@
+package tables
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20231130132931, Down_20231130132931)
+}
+
+func Up_20231130132931(tx *sql.Tx) error {
+	_, err := tx.Exec(`ALTER TABLE software ADD COLUMN title_id INT(10) UNSIGNED DEFAULT NULL`)
+	if err != nil {
+		return fmt.Errorf("failed to add title_id column to software table: %w", err)
+	}
+
+	_, err = tx.Exec(`ALTER TABLE software ADD INDEX (title_id)`)
+	if err != nil {
+		return fmt.Errorf("failed to add title_id index to software table: %w", err)
+	}
+
+	return nil
+}
+
+func Down_20231130132931(tx *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/migrations/tables/20231130132931_AlterSoftwareAddTitleId_test.go

@@ -0,0 +1,96 @@
+package tables
+
+import (
+	"context"
+	"testing"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUp_20231130132931(t *testing.T) {
+	db := applyUpToPrev(t)
+
+	insertStmt := "INSERT INTO software (name, source, version) VALUES (?, ?, ?)"
+
+	_, err := db.Exec(insertStmt, "test-name", "test-source", "test-version")
+	require.NoError(t, err)
+
+	_, err = db.Exec(insertStmt, "test-name2", "test-source", "test-version")
+	require.NoError(t, err)
+
+	_, err = db.Exec(insertStmt, "test-name", "test-source2", "test-version")
+	require.NoError(t, err)
+
+	_, err = db.Exec(insertStmt, "test-name", "test-source", "test-version2")
+	require.NoError(t, err)
+
+	// Apply current migration.
+	applyNext(t, db)
+
+	// Check that the title_id column was added.
+	selectStmt := `
+SELECT
+	id,
+	name,
+	version,
+	source,
+	title_id
+FROM software
+WHERE name IN ('test-name', 'test-name2') AND title_id IS NULL`
+
+	var rows []fleet.Software
+	err = sqlx.SelectContext(context.Background(), db, &rows, selectStmt)
+	require.NoError(t, err)
+	require.Len(t, rows, 4)
+
+	for _, row := range rows {
+		require.Contains(t, []string{"test-name", "test-name2"}, row.Name)
+		require.Contains(t, []string{"test-source", "test-source2"}, row.Source)
+		require.Contains(t, []string{"test-version", "test-version2"}, row.Version)
+		require.Nil(t, row.TitleID)
+	}
+
+	// add a row without the title_id set
+	_, err = db.Exec(insertStmt, "test-name", "test-source", "test-version3")
+	require.NoError(t, err)
+
+	// add a row with the title_id set
+	insertStmt = "INSERT INTO software (name, source, version, title_id) VALUES (?, ?, ?, ?)"
+	_, err = db.Exec(insertStmt, "test-name", "test-source", "test-version4", 1)
+	require.NoError(t, err)
+
+	selectStmt = `
+SELECT
+	id,
+	name,
+	version,
+	source,
+	title_id
+FROM software
+WHERE title_id = ?`
+
+	rows = []fleet.Software{}
+	err = sqlx.SelectContext(context.Background(), db, &rows, selectStmt, 1)
+	require.NoError(t, err)
+	require.Len(t, rows, 1)
+
+	updateStmt := "UPDATE software SET title_id = ? WHERE name = ? AND source = ?"
+
+	_, err = db.Exec(updateStmt, 1, "test-name", "test-source")
+	require.NoError(t, err)
+
+	rows = []fleet.Software{}
+	err = sqlx.SelectContext(context.Background(), db, &rows, selectStmt, 1)
+	require.NoError(t, err)
+	require.Len(t, rows, 4)
+
+	for _, row := range rows {
+		require.NotNil(t, row.TitleID)
+		require.Equal(t, uint(1), *row.TitleID)
+		require.Equal(t, "test-name", row.Name)
+		require.Equal(t, "test-source", row.Source)
+		require.Contains(t, []string{"test-version", "test-version2", "test-version3", "test-version4"}, row.Version)
+	}
+}

server/fleet/software.go

@@ -76,6 +76,10 @@ type Software struct {
 	// corresponding host. Only filled when the software list is requested for
 	// a specific host (host_id is provided).
 	LastOpenedAt *time.Time `json:"last_opened_at,omitempty" db:"last_opened_at"`
+
+	// TitleID is the ID of the associated software title, representing a unique combination of name
+	// and source.
+	TitleID *uint `json:"-" db:"title_id"`
 }
 
 func (Software) AuthzType() string {