Initial commit with the lib version of my personal migration tool

.gitignore

@@ -0,0 +1,94 @@
+# Created by https://www.gitignore.io/api/go,linux,macos,windows
+# Edit at https://www.gitignore.io/?templates=go,linux,macos,windows
+
+### Go ###
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+### Go Patch ###
+/vendor/
+/Godeps/
+
+### Linux ###
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# End of https://www.gitignore.io/api/go,linux,macos,windows

go.mod

@@ -0,0 +1,9 @@
+module github.com/joncalhoun/migrate
+
+go 1.14
+
+require (
+	github.com/jmoiron/sqlx v1.2.0
+	github.com/mattn/go-sqlite3 v2.0.3+incompatible
+	google.golang.org/appengine v1.6.6 // indirect
+)

go.sum

@@ -0,0 +1,18 @@
+github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
+github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=

sqlx.go

@@ -0,0 +1,136 @@
+package migrate
+
+import (
+	"database/sql"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/jmoiron/sqlx"
+)
+
+// Sqlx is a migrator that uses github.com/jmoiron/sqlx
+type Sqlx struct {
+	Migrations []SqlxMigration
+	// Printf is used to print out additional information during a migration, such
+	// as which step the migration is currently on. It can be replaced with any
+	// custom printf function, including one that just ignores inputs. If nil it
+	// will default to fmt.Printf.
+	Printf func(format string, a ...interface{}) (n int, err error)
+}
+
+// Migrate will run the migrations using the provided db connection.
+func (s *Sqlx) Migrate(sqlDB *sql.DB, dialect string) error {
+	db := sqlx.NewDb(sqlDB, dialect)
+
+	s.printf("Creating/checking migrations table...\n")
+	err := s.createMigrationTable(db)
+	if err != nil {
+		return err
+	}
+	for _, m := range s.Migrations {
+		var found string
+		err := db.Get(&found, "SELECT id FROM migrations WHERE id=$1", m.ID)
+		switch err {
+		case sql.ErrNoRows:
+			s.printf("Running migration: %v\n", m.ID)
+			// we need to run the migration so we continue to code below
+		case nil:
+			s.printf("Skipping migration: %v\n", m.ID)
+			continue
+		default:
+			return fmt.Errorf("looking up migration by id: %w", err)
+		}
+		err = s.runMigration(db, m)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Sqlx) printf(format string, a ...interface{}) (n int, err error) {
+	printf := s.Printf
+	if printf == nil {
+		printf = fmt.Printf
+	}
+	return printf(format, a...)
+}
+
+func (s *Sqlx) createMigrationTable(db *sqlx.DB) error {
+	_, err := db.Exec("CREATE TABLE IF NOT EXISTS migrations (id TEXT PRIMARY KEY )")
+	if err != nil {
+		return fmt.Errorf("creating migrations table: %w", err)
+	}
+	return nil
+}
+
+func (s *Sqlx) runMigration(db *sqlx.DB, m SqlxMigration) error {
+	errorf := func(err error) error { return fmt.Errorf("running migration: %w", err) }
+
+	tx, err := db.Beginx()
+	if err != nil {
+		return errorf(err)
+	}
+	_, err = db.Exec("INSERT INTO migrations (id) VALUES ($1)", m.ID)
+	if err != nil {
+		tx.Rollback()
+		return errorf(err)
+	}
+	err = m.Migrate(tx)
+	if err != nil {
+		tx.Rollback()
+		return errorf(err)
+	}
+	err = tx.Commit()
+	if err != nil {
+		return errorf(err)
+	}
+	return nil
+}
+
+// SqlxMigration is a unique ID plus a function that uses a sqlx transaction
+// to perform a database migration step.
+//
+// Note: Long term this could have a Rollback field if we wanted to support
+// that.
+type SqlxMigration struct {
+	ID      string
+	Migrate func(tx *sqlx.Tx) error
+}
+
+// SqlxQueryMigration will create a SqlxMigration using the provided id and
+// query string. It is a helper function designed to simplify the process of
+// creating migrations that only depending on a SQL query string.
+func SqlxQueryMigration(id, query string) SqlxMigration {
+	m := SqlxMigration{
+		ID: id,
+		Migrate: func(tx *sqlx.Tx) error {
+			_, err := tx.Exec(query)
+			return err
+		},
+	}
+	return m
+}
+
+// SqlxFileMigration will create a SqlxMigration using the provided file.
+func SqlxFileMigration(id, filename string) SqlxMigration {
+	f, err := os.Open(filename)
+	if err != nil {
+		// We could return a migration that errors when the migration is run, but I
+		// think it makes more sense to panic here.
+		panic(err)
+	}
+	fileBytes, err := ioutil.ReadAll(f)
+	if err != nil {
+		panic(err)
+	}
+	m := SqlxMigration{
+		ID: id,
+		Migrate: func(tx *sqlx.Tx) error {
+			_, err := tx.Exec(string(fileBytes))
+			return err
+		},
+	}
+	return m
+}

sqlx_test.go

@@ -0,0 +1,122 @@
+package migrate_test
+
+import (
+	"database/sql"
+	"fmt"
+	"testing"
+
+	"github.com/joncalhoun/migrate"
+	_ "github.com/mattn/go-sqlite3"
+)
+
+func sqliteInMem(t *testing.T) *sql.DB {
+	db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name()))
+	if err != nil {
+		t.Fatalf("Open() err = %v; want nil", err)
+	}
+	t.Cleanup(func() {
+		err = db.Close()
+		if err != nil {
+			t.Errorf("Close() err = %v; want nil", err)
+		}
+	})
+	return db
+}
+
+// TODO: Add more exhaustive testing. Perhaps try different dialects? Good enough for now tho.
+func TestSqlx(t *testing.T) {
+	t.Run("simple", func(t *testing.T) {
+		db := sqliteInMem(t)
+		migrator := migrate.Sqlx{
+			Printf: func(format string, args ...interface{}) (int, error) {
+				t.Logf(format, args...)
+				return 0, nil
+			},
+			Migrations: []migrate.SqlxMigration{
+				migrate.SqlxQueryMigration("001_create_courses", createCoursesSql),
+			},
+		}
+		err := migrator.Migrate(db, "sqlite3")
+		if err != nil {
+			t.Fatalf("Migrate() err = %v; want nil", err)
+		}
+		_, err = db.Exec("INSERT INTO courses (name) VALUES ($1) ", "cor_test")
+		if err != nil {
+			t.Fatalf("db.Exec() err = %v; want nil", err)
+		}
+	})
+
+	t.Run("existing migrations", func(t *testing.T) {
+		db := sqliteInMem(t)
+		migrator := migrate.Sqlx{
+			Printf: func(format string, args ...interface{}) (int, error) {
+				t.Logf(format, args...)
+				return 0, nil
+			},
+			Migrations: []migrate.SqlxMigration{
+				migrate.SqlxQueryMigration("001_create_courses", createCoursesSql),
+			},
+		}
+		err := migrator.Migrate(db, "sqlite3")
+		if err != nil {
+			t.Fatalf("Migrate() err = %v; want nil", err)
+		}
+		_, err = db.Exec("INSERT INTO courses (name) VALUES ($1) ", "cor_test")
+		if err != nil {
+			t.Fatalf("db.Exec() err = %v; want nil", err)
+		}
+
+		// the real test
+		migrator = migrate.Sqlx{
+			Printf: func(format string, args ...interface{}) (int, error) {
+				t.Logf(format, args...)
+				return 0, nil
+			},
+			Migrations: []migrate.SqlxMigration{
+				migrate.SqlxQueryMigration("001_create_courses", createCoursesSql),
+				migrate.SqlxQueryMigration("002_create_users", createUsersSql),
+			},
+		}
+		err = migrator.Migrate(db, "sqlite3")
+		if err != nil {
+			t.Fatalf("Migrate() err = %v; want nil", err)
+		}
+		_, err = db.Exec("INSERT INTO users (email) VALUES ($1) ", "abc@test.com")
+		if err != nil {
+			t.Fatalf("db.Exec() err = %v; want nil", err)
+		}
+	})
+
+	t.Run("file", func(t *testing.T) {
+		db := sqliteInMem(t)
+		migrator := migrate.Sqlx{
+			Printf: func(format string, args ...interface{}) (int, error) {
+				t.Logf(format, args...)
+				return 0, nil
+			},
+			Migrations: []migrate.SqlxMigration{
+				migrate.SqlxFileMigration("001_create_widgets", "testdata/widgets.sql"),
+			},
+		}
+		err := migrator.Migrate(db, "sqlite3")
+		if err != nil {
+			t.Fatalf("Migrate() err = %v; want nil", err)
+		}
+		_, err = db.Exec("INSERT INTO widgets (color, price) VALUES ($1, $2)", "red", 1200)
+		if err != nil {
+			t.Fatalf("db.Exec() err = %v; want nil", err)
+		}
+	})
+}
+
+var createCoursesSql = `
+CREATE TABLE courses (
+  id serial PRIMARY KEY,
+  name text
+);`
+
+var createUsersSql = `
+CREATE TABLE users (
+  id serial PRIMARY KEY,
+  email text UNIQUE NOT NULL
+);`

testdata/widgets.sql

@@ -0,0 +1,6 @@
+CREATE TABLE widgets (
+  id serial PRIMARY KEY,
+  color text NOT NULL,
+  price integer
+);
+