This repository has been archived by the owner on Mar 11, 2021. It is now read-only.
/
migration.go
444 lines (342 loc) · 13 KB
/
migration.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
package migration
import (
"bufio"
"bytes"
"context"
"database/sql"
"net/http"
"net/url"
"sync"
"text/template"
"github.com/fabric8-services/fabric8-auth/log"
"github.com/goadesign/goa"
"github.com/goadesign/goa/client"
errs "github.com/pkg/errors"
)
// AdvisoryLockID is a random number that should be used within the application
// by anybody who wants to modify the "version" table.
const AdvisoryLockID = 42
// fn defines the type of function that can be part of a migration steps
type fn func(tx *sql.Tx) error
// steps defines a collection of all the functions that make up a version
type steps []fn
// Migrations defines all a collection of all the steps
type Migrations []steps
// mutex variable to lock/unlock the population of common types
var populateLocker = &sync.Mutex{}
type MigrationConfiguration interface {
GetOpenShiftClientApiUrl() string
}
// Migrate executes the required migration of the database on startup.
// For each successful migration, an entry will be written into the "version"
// table, that states when a certain version was reached.
func Migrate(db *sql.DB, catalog string, configuration MigrationConfiguration) error {
var err error
if db == nil {
return errs.Errorf("Database handle is nil\n")
}
m := GetMigrations(configuration)
var tx *sql.Tx
for nextVersion := int64(0); nextVersion < int64(len(m)) && err == nil; nextVersion++ {
tx, err = db.Begin()
if err != nil {
return errs.Errorf("Failed to start transaction: %s\n", err)
}
err = MigrateToNextVersion(tx, &nextVersion, m, catalog)
if err != nil {
oldErr := err
log.Info(nil, map[string]interface{}{
"next_version": nextVersion,
"migrations": m,
"err": err,
}, "Rolling back transaction due to: %v", err)
if err = tx.Rollback(); err != nil {
log.Error(nil, map[string]interface{}{
"next_version": nextVersion,
"migrations": m,
"err": err,
}, "error while rolling back transaction")
return errs.Errorf("Error while rolling back transaction: %s\n", err)
}
return oldErr
}
if err = tx.Commit(); err != nil {
log.Error(nil, map[string]interface{}{
"migrations": m,
"err": err,
}, "error during transaction commit: %v", err)
return errs.Errorf("Error during transaction commit: %s\n", err)
}
}
if err != nil {
log.Error(nil, map[string]interface{}{
"migrations": m,
"err": err,
}, "migration failed with error: %v", err)
return errs.Errorf("Migration failed with error: %s\n", err)
}
return nil
}
// GetMigrations returns the migrations all the migrations we have.
// Add your own migration to the end of this function.
// IMPORTANT: ALWAYS APPEND AT THE END AND DON'T CHANGE THE ORDER OF MIGRATIONS!
func GetMigrations(configuration MigrationConfiguration) Migrations {
m := Migrations{}
// Version 0
m = append(m, steps{ExecuteSQLFile("000-bootstrap.sql")})
// Version 1
m = append(m, steps{ExecuteSQLFile("001-identities-users.sql")})
// Version 2
m = append(m, steps{ExecuteSQLFile("002-oauth-states.sql")})
// Version 3
m = append(m, steps{ExecuteSQLFile("003-space-resources.sql")})
// Version 4
m = append(m, steps{ExecuteSQLFile("004-unique-resource-space.sql")})
// Version 5
m = append(m, steps{ExecuteSQLFile("005-authorization.sql")})
// Version 6
m = append(m, steps{ExecuteSQLFile("006-external-provider.sql")})
// Version 7
m = append(m, steps{ExecuteSQLFile("007-external-provider-id-index.sql")})
// Version 8
m = append(m, steps{ExecuteSQLFile("008-rename-token-table.sql")})
// Version 9
m = append(m, steps{ExecuteSQLFile("009-external-token-hard-delete.sql")})
// Version 10
defaultCluster := configuration.GetOpenShiftClientApiUrl()
m = append(m, steps{ExecuteSQLFile("010-add-cluster-to-user.sql", defaultCluster)})
// Version 11
m = append(m, steps{ExecuteSQLFile("011-add-username-to-external-token.sql")})
// Version 12
m = append(m, steps{ExecuteSQLFile("012-hide-email.sql")})
// Version 13
m = append(m, steps{ExecuteSQLFile("013-add-email-verified.sql")})
// Version 14
m = append(m, steps{ExecuteSQLFile("014-add-user-feature-level.sql")})
// Version 15
m = append(m, steps{ExecuteSQLFile("015-clear-resources-create-resource-types.sql")})
// Version 16
m = append(m, steps{ExecuteSQLFile("016-add-state-to-auth-state-reference.sql")})
// Version 17
m = append(m, steps{ExecuteSQLFile("017-feature-level-not-null.sql")})
// Version 18
m = append(m, steps{ExecuteSQLFile("018-convert-user-feature-level.sql")})
// Version 19
m = append(m, steps{ExecuteSQLFile("019-authorization-part-2.sql")})
// Version 20
m = append(m, steps{ExecuteSQLFile("020-add-response-mode-to-auth-state-reference.sql")})
// Version 21
m = append(m, steps{ExecuteSQLFile("021-organizations-list-create.sql")})
// Version 22
m = append(m, steps{ExecuteSQLFile("022-add-deprovisioned-to-user.sql")})
// Version 23
m = append(m, steps{ExecuteSQLFile("023-resource-type-index.sql")})
// Version 24
m = append(m, steps{ExecuteSQLFile("024-role-mapping-and-team-and-group-identities.sql")})
// Version 25
m = append(m, steps{ExecuteSQLFile("025-fix-feature-level.sql")})
// Version 26
m = append(m, steps{ExecuteSQLFile("026-identities-users-indexes.sql")})
// Version 27
m = append(m, steps{ExecuteSQLFile("027-invitations.sql")})
// Version 28
m = append(m, steps{ExecuteSQLFile("028-make-organization-names-unique.sql")})
// Version 29
m = append(m, steps{ExecuteSQLFile("029-add-space-resourcetype.sql")})
// Version 30
m = append(m, steps{ExecuteSQLFile("030-add-team-admin-role.sql")})
// Version 31
m = append(m, steps{ExecuteSQLFile("031-clean-up-roles-scopes.sql")})
// Version 32
m = append(m, steps{ExecuteSQLFile("032-invitation-code.sql")})
// Version 33
m = append(m, steps{ExecuteSQLFile("033-drop-space-resources.sql")})
// Version 34
m = append(m, steps{ExecuteSQLFile("034-rename-token-table.sql")})
// Version 35
m = append(m, steps{ExecuteSQLFile("035-unique_constraint_default_role_mapping.sql")})
// Version 36
m = append(m, steps{ExecuteSQLFile("036-token-privileges.sql")})
// Version 37
m = append(m, steps{ExecuteSQLFile("037-invitation-redirect-url.sql")})
// Version 38
m = append(m, steps{ExecuteSQLFile("038-admin-console-resource.sql")})
// Version 39
m = append(m, steps{ExecuteSQLFile("039-resource-type-alter.sql")})
// Version 40
m = append(m, steps{ExecuteSQLFile("040-deferrable-constraints.sql")})
// Version 41
m = append(m, steps{ExecuteSQLFile("041-identity-role-index.sql")})
// Version 42
m = append(m, steps{ExecuteSQLFile("042-token-index.sql")})
// Version 43
m = append(m, steps{ExecuteSQLFile("043-add-admin-console-resource.sql")})
// Version 44
m = append(m, steps{ExecuteSQLFile("044-user-active.sql")})
// Version 45
m = append(m, steps{ExecuteSQLFile("045-identity-last-active.sql")})
// Version 46
m = append(m, steps{ExecuteSQLFile("046-identity-last-active-default.sql")})
// Version 47
m = append(m, steps{ExecuteSQLFile("047-add-user-banned-column.sql")})
// Version 48
m = append(m, steps{ExecuteSQLFile("048-identity-deactivation-notification.sql")})
// Version 49
m = append(m, steps{ExecuteSQLFile("049-user-banned-index.sql")})
// Version 50
m = append(m, steps{ExecuteSQLFile("050-worker-lock.sql")})
// Version 51
m = append(m, steps{ExecuteSQLFile("051-identity-deactivation-scheduled.sql")})
// Version 52
m = append(m, steps{ExecuteSQLFile("052-deferrable-constraints2.sql")})
// Version 53
m = append(m, steps{ExecuteSQLFile("053-deactivation-indexes.sql")})
// Version 54
m = append(m, steps{ExecuteSQLFile("054-cleanup-oauth-state-references.sql")})
// Version N
//
// In order to add an upgrade, simply append an array of MigrationFunc to the
// the end of the "migrations" slice. The version numbers are determined by
// the index in the array. The following code in comments show how you can
// do a migration in 3 steps. If one of the steps fails, the others are not
// executed.
// If something goes wrong during the migration, all you need to do is return
// an error that is not nil.
/*
m = append(m, steps{
func(db *sql.Tx) error {
// Execute random go code
return nil
},
ExecuteSQLFile("YOUR_OWN_FILE.sql"),
func(db *sql.Tx) error {
// Execute random go code
return nil
},
})
*/
return m
}
// ExecuteSQLFile loads the given filename from the packaged SQL files and
// executes it on the given database. Golang text/template module is used
// to handle all the optional arguments passed to the sql files
func ExecuteSQLFile(filename string, args ...string) fn {
return func(db *sql.Tx) error {
data, err := Asset(filename)
if err != nil {
return errs.Wrapf(err, "failed to find filename: %s", filename)
}
if len(args) > 0 {
tmpl, err := template.New("sql").Parse(string(data))
if err != nil {
return errs.Wrap(err, "failed to parse SQL template")
}
var sqlScript bytes.Buffer
writer := bufio.NewWriter(&sqlScript)
err = tmpl.Execute(writer, args)
if err != nil {
return errs.Wrap(err, "failed to execute SQL template")
}
// We need to flush the content of the writer
writer.Flush()
_, err = db.Exec(sqlScript.String())
if err != nil {
log.Error(context.Background(), map[string]interface{}{
"err": err,
}, "failed to execute this query: \n\n%s\n\n", sqlScript.String())
}
} else {
_, err = db.Exec(string(data))
if err != nil {
log.Error(context.Background(), map[string]interface{}{
"err": err,
}, "failed to execute this query: \n\n%s\n\n", string(data))
}
}
return errs.WithStack(err)
}
}
// MigrateToNextVersion migrates the database to the nextVersion.
// If the database is already at nextVersion or higher, the nextVersion
// will be set to the actual next version.
func MigrateToNextVersion(tx *sql.Tx, nextVersion *int64, m Migrations, catalog string) error {
// Obtain exclusive transaction level advisory that doesn't depend on any table.
// Once obtained, the lock is held for the remainder of the current transaction.
// (There is no UNLOCK TABLE command; locks are always released at transaction end.)
if _, err := tx.Exec("SELECT pg_advisory_xact_lock($1)", AdvisoryLockID); err != nil {
return errs.Errorf("Failed to acquire lock: %s\n", err)
}
// Determine current version and adjust the outmost loop
// iterator variable "version"
currentVersion, err := getCurrentVersion(tx, catalog)
if err != nil {
return errs.WithStack(err)
}
*nextVersion = currentVersion + 1
if *nextVersion >= int64(len(m)) {
// No further updates to apply (this is NOT an error)
log.Info(nil, map[string]interface{}{
"next_version": *nextVersion,
"current_version": currentVersion,
}, "Current version %d. Nothing to update.", currentVersion)
return nil
}
log.Info(nil, map[string]interface{}{
"next_version": *nextVersion,
"current_version": currentVersion,
}, "Attempt to update DB to version %v", *nextVersion)
// Apply all the updates of the next version
for j := range m[*nextVersion] {
if err := m[*nextVersion][j](tx); err != nil {
return errs.Errorf("Failed to execute migration of step %d of version %d: %s\n", j, *nextVersion, err)
}
}
if _, err := tx.Exec("INSERT INTO version(version) VALUES($1)", *nextVersion); err != nil {
return errs.Errorf("Failed to update DB to version %d: %s\n", *nextVersion, err)
}
log.Info(nil, map[string]interface{}{
"next_version": *nextVersion,
"current_version": currentVersion,
}, "Successfully updated DB to version %v", *nextVersion)
return nil
}
// getCurrentVersion returns the highest version from the version
// table or -1 if that table does not exist.
//
// Returning -1 simplifies the logic of the migration process because
// the next version is always the current version + 1 which results
// in -1 + 1 = 0 which is exactly what we want as the first version.
func getCurrentVersion(db *sql.Tx, catalog string) (int64, error) {
query := `SELECT EXISTS(
SELECT 1 FROM information_schema.tables
WHERE table_catalog=$1
AND table_name='version')`
row := db.QueryRow(query, catalog)
var exists bool
if err := row.Scan(&exists); err != nil {
return -1, errs.Errorf("Failed to scan if table \"version\" exists: %s\n", err)
}
if !exists {
// table doesn't exist
return -1, nil
}
row = db.QueryRow("SELECT max(version) as current FROM version")
var current int64 = -1
if err := row.Scan(¤t); err != nil {
return -1, errs.Errorf("Failed to scan max version in table \"version\": %s\n", err)
}
return current, nil
}
// NewMigrationContext aims to create a new goa context where to initialize the
// request and req_id context keys.
// NOTE: We need this function to initialize the goa.ContextRequest
func NewMigrationContext(ctx context.Context) context.Context {
req := &http.Request{Host: "localhost"}
params := url.Values{}
ctx = goa.NewContext(ctx, nil, req, params)
// set a random request ID for the context
var reqID string
ctx, reqID = client.ContextWithRequestID(ctx)
log.Debug(ctx, nil, "Initialized the migration context with Request ID: %v", reqID)
return ctx
}