From 006e7c7cc36b94a31c6e931f017785cabcf00e3e Mon Sep 17 00:00:00 2001 From: Jon Calhoun Date: Thu, 28 May 2020 13:06:53 -0400 Subject: [PATCH] Initial commit with the lib version of my personal migration tool --- .gitignore | 94 ++++++++++++++++++++++++++++++ go.mod | 9 +++ go.sum | 18 ++++++ sqlx.go | 136 +++++++++++++++++++++++++++++++++++++++++++ sqlx_test.go | 122 ++++++++++++++++++++++++++++++++++++++ testdata/widgets.sql | 6 ++ 6 files changed, 385 insertions(+) create mode 100644 .gitignore create mode 100644 go.mod create mode 100644 go.sum create mode 100644 sqlx.go create mode 100644 sqlx_test.go create mode 100644 testdata/widgets.sql diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..14a289e --- /dev/null +++ b/.gitignore @@ -0,0 +1,94 @@ +# Created by https://www.gitignore.io/api/go,linux,macos,windows +# Edit at https://www.gitignore.io/?templates=go,linux,macos,windows + +### Go ### +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +### Go Patch ### +/vendor/ +/Godeps/ + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.gitignore.io/api/go,linux,macos,windows diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b5167a9 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/joncalhoun/migrate + +go 1.14 + +require ( + github.com/jmoiron/sqlx v1.2.0 + github.com/mattn/go-sqlite3 v2.0.3+incompatible + google.golang.org/appengine v1.6.6 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..dfe06b6 --- /dev/null +++ b/go.sum @@ -0,0 +1,18 @@ +github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= +github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= diff --git a/sqlx.go b/sqlx.go new file mode 100644 index 0000000..e74b687 --- /dev/null +++ b/sqlx.go @@ -0,0 +1,136 @@ +package migrate + +import ( + "database/sql" + "fmt" + "io/ioutil" + "os" + + "github.com/jmoiron/sqlx" +) + +// Sqlx is a migrator that uses github.com/jmoiron/sqlx +type Sqlx struct { + Migrations []SqlxMigration + // Printf is used to print out additional information during a migration, such + // as which step the migration is currently on. It can be replaced with any + // custom printf function, including one that just ignores inputs. If nil it + // will default to fmt.Printf. + Printf func(format string, a ...interface{}) (n int, err error) +} + +// Migrate will run the migrations using the provided db connection. +func (s *Sqlx) Migrate(sqlDB *sql.DB, dialect string) error { + db := sqlx.NewDb(sqlDB, dialect) + + s.printf("Creating/checking migrations table...\n") + err := s.createMigrationTable(db) + if err != nil { + return err + } + for _, m := range s.Migrations { + var found string + err := db.Get(&found, "SELECT id FROM migrations WHERE id=$1", m.ID) + switch err { + case sql.ErrNoRows: + s.printf("Running migration: %v\n", m.ID) + // we need to run the migration so we continue to code below + case nil: + s.printf("Skipping migration: %v\n", m.ID) + continue + default: + return fmt.Errorf("looking up migration by id: %w", err) + } + err = s.runMigration(db, m) + if err != nil { + return err + } + } + return nil +} + +func (s *Sqlx) printf(format string, a ...interface{}) (n int, err error) { + printf := s.Printf + if printf == nil { + printf = fmt.Printf + } + return printf(format, a...) +} + +func (s *Sqlx) createMigrationTable(db *sqlx.DB) error { + _, err := db.Exec("CREATE TABLE IF NOT EXISTS migrations (id TEXT PRIMARY KEY )") + if err != nil { + return fmt.Errorf("creating migrations table: %w", err) + } + return nil +} + +func (s *Sqlx) runMigration(db *sqlx.DB, m SqlxMigration) error { + errorf := func(err error) error { return fmt.Errorf("running migration: %w", err) } + + tx, err := db.Beginx() + if err != nil { + return errorf(err) + } + _, err = db.Exec("INSERT INTO migrations (id) VALUES ($1)", m.ID) + if err != nil { + tx.Rollback() + return errorf(err) + } + err = m.Migrate(tx) + if err != nil { + tx.Rollback() + return errorf(err) + } + err = tx.Commit() + if err != nil { + return errorf(err) + } + return nil +} + +// SqlxMigration is a unique ID plus a function that uses a sqlx transaction +// to perform a database migration step. +// +// Note: Long term this could have a Rollback field if we wanted to support +// that. +type SqlxMigration struct { + ID string + Migrate func(tx *sqlx.Tx) error +} + +// SqlxQueryMigration will create a SqlxMigration using the provided id and +// query string. It is a helper function designed to simplify the process of +// creating migrations that only depending on a SQL query string. +func SqlxQueryMigration(id, query string) SqlxMigration { + m := SqlxMigration{ + ID: id, + Migrate: func(tx *sqlx.Tx) error { + _, err := tx.Exec(query) + return err + }, + } + return m +} + +// SqlxFileMigration will create a SqlxMigration using the provided file. +func SqlxFileMigration(id, filename string) SqlxMigration { + f, err := os.Open(filename) + if err != nil { + // We could return a migration that errors when the migration is run, but I + // think it makes more sense to panic here. + panic(err) + } + fileBytes, err := ioutil.ReadAll(f) + if err != nil { + panic(err) + } + m := SqlxMigration{ + ID: id, + Migrate: func(tx *sqlx.Tx) error { + _, err := tx.Exec(string(fileBytes)) + return err + }, + } + return m +} diff --git a/sqlx_test.go b/sqlx_test.go new file mode 100644 index 0000000..2f377de --- /dev/null +++ b/sqlx_test.go @@ -0,0 +1,122 @@ +package migrate_test + +import ( + "database/sql" + "fmt" + "testing" + + "github.com/joncalhoun/migrate" + _ "github.com/mattn/go-sqlite3" +) + +func sqliteInMem(t *testing.T) *sql.DB { + db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())) + if err != nil { + t.Fatalf("Open() err = %v; want nil", err) + } + t.Cleanup(func() { + err = db.Close() + if err != nil { + t.Errorf("Close() err = %v; want nil", err) + } + }) + return db +} + +// TODO: Add more exhaustive testing. Perhaps try different dialects? Good enough for now tho. +func TestSqlx(t *testing.T) { + t.Run("simple", func(t *testing.T) { + db := sqliteInMem(t) + migrator := migrate.Sqlx{ + Printf: func(format string, args ...interface{}) (int, error) { + t.Logf(format, args...) + return 0, nil + }, + Migrations: []migrate.SqlxMigration{ + migrate.SqlxQueryMigration("001_create_courses", createCoursesSql), + }, + } + err := migrator.Migrate(db, "sqlite3") + if err != nil { + t.Fatalf("Migrate() err = %v; want nil", err) + } + _, err = db.Exec("INSERT INTO courses (name) VALUES ($1) ", "cor_test") + if err != nil { + t.Fatalf("db.Exec() err = %v; want nil", err) + } + }) + + t.Run("existing migrations", func(t *testing.T) { + db := sqliteInMem(t) + migrator := migrate.Sqlx{ + Printf: func(format string, args ...interface{}) (int, error) { + t.Logf(format, args...) + return 0, nil + }, + Migrations: []migrate.SqlxMigration{ + migrate.SqlxQueryMigration("001_create_courses", createCoursesSql), + }, + } + err := migrator.Migrate(db, "sqlite3") + if err != nil { + t.Fatalf("Migrate() err = %v; want nil", err) + } + _, err = db.Exec("INSERT INTO courses (name) VALUES ($1) ", "cor_test") + if err != nil { + t.Fatalf("db.Exec() err = %v; want nil", err) + } + + // the real test + migrator = migrate.Sqlx{ + Printf: func(format string, args ...interface{}) (int, error) { + t.Logf(format, args...) + return 0, nil + }, + Migrations: []migrate.SqlxMigration{ + migrate.SqlxQueryMigration("001_create_courses", createCoursesSql), + migrate.SqlxQueryMigration("002_create_users", createUsersSql), + }, + } + err = migrator.Migrate(db, "sqlite3") + if err != nil { + t.Fatalf("Migrate() err = %v; want nil", err) + } + _, err = db.Exec("INSERT INTO users (email) VALUES ($1) ", "abc@test.com") + if err != nil { + t.Fatalf("db.Exec() err = %v; want nil", err) + } + }) + + t.Run("file", func(t *testing.T) { + db := sqliteInMem(t) + migrator := migrate.Sqlx{ + Printf: func(format string, args ...interface{}) (int, error) { + t.Logf(format, args...) + return 0, nil + }, + Migrations: []migrate.SqlxMigration{ + migrate.SqlxFileMigration("001_create_widgets", "testdata/widgets.sql"), + }, + } + err := migrator.Migrate(db, "sqlite3") + if err != nil { + t.Fatalf("Migrate() err = %v; want nil", err) + } + _, err = db.Exec("INSERT INTO widgets (color, price) VALUES ($1, $2)", "red", 1200) + if err != nil { + t.Fatalf("db.Exec() err = %v; want nil", err) + } + }) +} + +var createCoursesSql = ` +CREATE TABLE courses ( + id serial PRIMARY KEY, + name text +);` + +var createUsersSql = ` +CREATE TABLE users ( + id serial PRIMARY KEY, + email text UNIQUE NOT NULL +);` diff --git a/testdata/widgets.sql b/testdata/widgets.sql new file mode 100644 index 0000000..d25fa84 --- /dev/null +++ b/testdata/widgets.sql @@ -0,0 +1,6 @@ +CREATE TABLE widgets ( + id serial PRIMARY KEY, + color text NOT NULL, + price integer +); +