From 10ab5684970c2d89b8ec3f324ab4d6e50b1b1e21 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Fri, 24 Sep 2021 13:30:21 +0800 Subject: [PATCH 01/23] support lightning Signed-off-by: Little-Wallace --- cmd/dm-worker/main.go | 2 + dm/config/subtask.go | 44 +++++- dm/config/task.go | 3 + dm/master/server.go | 5 + dm/worker/subtask.go | 6 +- go.mod | 4 +- go.sum | 17 ++- loader/checkpoint.go | 2 +- loader/lightning.go | 319 +++++++++++++++++++++++++++++++++++++++ pkg/schema/tracker.go | 2 +- pkg/terror/error_list.go | 2 + 11 files changed, 394 insertions(+), 12 deletions(-) create mode 100644 loader/lightning.go diff --git a/cmd/dm-worker/main.go b/cmd/dm-worker/main.go index 9187ec9b09..220c354b59 100644 --- a/cmd/dm-worker/main.go +++ b/cmd/dm-worker/main.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/terror" "github.com/pingcap/dm/pkg/utils" + lightningLog "github.com/pingcap/tidb/br/pkg/lightning/log" ) func main() { @@ -53,6 +54,7 @@ func main() { common.PrintLinesf("init logger error %s", terror.Message(err)) os.Exit(2) } + lightningLog.SetAppLogger(log.L().Logger) utils.LogHTTPProxies(true) diff --git a/dm/config/subtask.go b/dm/config/subtask.go index b882626bb7..013693eaab 100644 --- a/dm/config/subtask.go +++ b/dm/config/subtask.go @@ -23,6 +23,8 @@ import ( "strconv" "strings" + lcfg "github.com/pingcap/tidb/br/pkg/lightning/config" + "github.com/BurntSushi/toml" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" "github.com/pingcap/tidb-tools/pkg/column-mapping" @@ -184,6 +186,37 @@ func GetDBConfigFromEnv() DBConfig { } } +// TiDBExtraConfig is the extra DB configuration only for TiDb. +type TiDBExtraConfig struct { + StatusPort int `toml:"status-port" json:"status-port" yaml:"status-port"` + PdAddr string `toml:"pd-addr" json:"pd-addr" yaml:"pd-addr"` + Backend string `toml:"backend" json:"backend" yaml:"backend"` +} + +func (db *TiDBExtraConfig) String() string { + cfg, err := json.Marshal(db) + if err != nil { + log.L().Error("fail to marshal config to json", log.ShortError(err)) + } + return string(cfg) +} + +// Toml returns TOML format representation of config. +func (db *TiDBExtraConfig) Toml() (string, error) { + var b bytes.Buffer + enc := toml.NewEncoder(&b) + if err := enc.Encode(db); err != nil { + return "", terror.ErrConfigTomlTransform.Delegate(err, "encode db config to toml") + } + return b.String(), nil +} + +// Decode loads config from file data. +func (db *TiDBExtraConfig) Decode(data string) error { + _, err := toml.Decode(data, db) + return terror.ErrConfigTomlTransform.Delegate(err, "decode db config") +} + // SubTaskConfig is the configuration for SubTask. type SubTaskConfig struct { // BurntSushi/toml seems have a bug for flag "-" @@ -232,9 +265,10 @@ type SubTaskConfig struct { RelayDir string `toml:"relay-dir" json:"relay-dir"` // UseRelay get value from dm-worker's relayEnabled - UseRelay bool `toml:"use-relay" json:"use-relay"` - From DBConfig `toml:"from" json:"from"` - To DBConfig `toml:"to" json:"to"` + UseRelay bool `toml:"use-relay" json:"use-relay"` + From DBConfig `toml:"from" json:"from"` + To DBConfig `toml:"to" json:"to"` + TiDB TiDBExtraConfig `toml:"tidb" json:"tidb"` RouteRules []*router.TableRule `toml:"route-rules" json:"route-rules"` FilterRules []*bf.BinlogEventRule `toml:"filter-rules" json:"filter-rules"` @@ -446,6 +480,10 @@ func (c *SubTaskConfig) Adjust(verifyDecryptPassword bool) error { return terror.ErrConfigInvalidChunkFileSize.Generate(c.MydumperConfig.ChunkFilesize) } + if c.TiDB.Backend != "" && c.TiDB.Backend != lcfg.BackendLocal && c.TiDB.Backend != lcfg.BackendTiDB { + return terror.ErrLoadBackendNotSupport.Generate(c.TiDB.Backend) + } + // TODO: check every member // TODO: since we checked here, we could remove other terror like ErrSyncerUnitGenBAList // TODO: or we should check at task config and source config rather than this subtask config, to reduce duplication diff --git a/dm/config/task.go b/dm/config/task.go index 2c08c6e093..c4cc229f80 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -328,6 +328,9 @@ type TaskConfig struct { // deprecated, replaced by `start-task --remove-meta` RemoveMeta bool `yaml:"remove-meta"` + + // extra config when target db is TiDB + TiDB *TiDBExtraConfig `yaml:"tidb" toml:"tidb" json:"tidb"` } // NewTaskConfig creates a TaskConfig. diff --git a/dm/master/server.go b/dm/master/server.go index 11c5d48ecc..172359a924 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -1373,6 +1373,11 @@ func (s *Server) generateSubTask(ctx context.Context, task string, errCnt, warnC sourceCfgs := s.getSourceConfigs(cfg.MySQLInstances) stCfgs, err := cfg.SubTaskConfigs(sourceCfgs) + if cfg.TiDB != nil { + for _, st := range stCfgs { + st.TiDB = *cfg.TiDB + } + } if err != nil { return nil, nil, terror.WithClass(err, terror.ClassDMMaster) } diff --git a/dm/worker/subtask.go b/dm/worker/subtask.go index 037435cbb0..cedda0f7ec 100644 --- a/dm/worker/subtask.go +++ b/dm/worker/subtask.go @@ -59,7 +59,11 @@ func createRealUnits(cfg *config.SubTaskConfig, etcdClient *clientv3.Client, wor switch cfg.Mode { case config.ModeAll: us = append(us, dumpling.NewDumpling(cfg)) - us = append(us, loader.NewLoader(cfg, etcdClient, workerName)) + if cfg.TiDB.Backend == "" { + us = append(us, loader.NewLoader(cfg, etcdClient, workerName)) + } else { + us = append(us, loader.NewLightningLoader(cfg, etcdClient, workerName)) + } us = append(us, syncer.NewSyncer(cfg, etcdClient)) case config.ModeFull: // NOTE: maybe need another checker in the future? diff --git a/go.mod b/go.mod index 2c2eb0b00c..ead1b92207 100644 --- a/go.mod +++ b/go.mod @@ -25,8 +25,8 @@ require ( github.com/pingcap/errors v0.11.5-0.20210513014640-40f9a1999b3b github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 - github.com/pingcap/parser v0.0.0-20210907051057-948434fa20e4 - github.com/pingcap/tidb v1.1.0-beta.0.20210914112841-6ebfe8aa4257 + github.com/pingcap/parser v0.0.0-20210915160042-658edc1dba6a + github.com/pingcap/tidb v1.1.0-beta.0.20210923162246-d14566c7ed3e github.com/pingcap/tidb-tools v5.2.0-alpha.0.20210727084616-915b22e4d42c+incompatible github.com/prometheus/client_golang v1.5.1 github.com/rakyll/statik v0.1.6 diff --git a/go.sum b/go.sum index 2a187219ea..93f1836351 100644 --- a/go.sum +++ b/go.sum @@ -51,6 +51,7 @@ github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EF github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgRRqzCuPshRkQ7I= github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= @@ -131,6 +132,7 @@ github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQ github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20210719141320-8c3bd06debb5 h1:Igd6YmtOZ77EgLAIaE9+mHl7+sAKaZ5m4iMI0Dz/J2A= github.com/cockroachdb/pebble v0.0.0-20210719141320-8c3bd06debb5/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= @@ -236,6 +238,7 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-echarts/go-echarts v1.0.0/go.mod h1:qbmyAb/Rl1f2w7wKba1D4LoNq4U164yO4/wedFbcWyo= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -430,6 +433,7 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jedib0t/go-pretty/v6 v6.2.2 h1:o3McN0rQ4X+IU+HduppSp9TwRdGLRW2rhJXy9CJaCRw= github.com/jedib0t/go-pretty/v6 v6.2.2/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= @@ -637,15 +641,17 @@ github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 h1:SvWCbCPh1YeHd9yQLks github.com/pingcap/log v0.0.0-20210906054005-afc726e70354/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/parser v0.0.0-20210415081931-48e7f467fd74/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= -github.com/pingcap/parser v0.0.0-20210907051057-948434fa20e4 h1:v3paTSRJgM7E/2CqIxKd5gNSrlknktjIFNS9NMbpFLo= github.com/pingcap/parser v0.0.0-20210907051057-948434fa20e4/go.mod h1:Ek0mLKEqUGnQqBw1JnYrJQxsguU433DU68yUbsoeJ7s= +github.com/pingcap/parser v0.0.0-20210915160042-658edc1dba6a h1:dApxKqk49ZaE/oErh4h5W27SJyyy5/oywIzpStANgLI= +github.com/pingcap/parser v0.0.0-20210915160042-658edc1dba6a/go.mod h1:+xcMiiZzdIktT/Nqdfm81dkECJ2EPuoAYywd57py4Pk= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3/go.mod h1:tckvA041UWP+NqYzrJ3fMgC/Hw9wnmQ/tUkp/JaHly8= github.com/pingcap/sysutil v0.0.0-20210730114356-fcd8a63f68c5 h1:7rvAtZe/ZUzOKzgriNPQoBNvleJXBk4z7L3Z47+tS98= github.com/pingcap/sysutil v0.0.0-20210730114356-fcd8a63f68c5/go.mod h1:XsOaV712rUk63aOEKYP9PhXTIE3FMNHmC2r1wX5wElY= github.com/pingcap/tidb v1.1.0-beta.0.20210914080841-89617e20b2c0/go.mod h1:MSg0d2+iJI/d3mUOcLV82GLpw5+jNGq21Z8qcHqBI/g= -github.com/pingcap/tidb v1.1.0-beta.0.20210914112841-6ebfe8aa4257 h1:5VoUNB3w5OX1vAuHfYWnsdt4brCmoYcljPqcGjvZc0E= github.com/pingcap/tidb v1.1.0-beta.0.20210914112841-6ebfe8aa4257/go.mod h1:MSg0d2+iJI/d3mUOcLV82GLpw5+jNGq21Z8qcHqBI/g= +github.com/pingcap/tidb v1.1.0-beta.0.20210923162246-d14566c7ed3e h1:2MFgYfLNHoEMhUw0Nx6gRhh+qK7rADQl5V2aTTDc088= +github.com/pingcap/tidb v1.1.0-beta.0.20210923162246-d14566c7ed3e/go.mod h1:EWxcIyryh3CTwyuzlqllMZhrgxFbjJkUmx9/UXN0GVE= github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo= github.com/pingcap/tidb-dashboard v0.0.0-20210716172320-2226872e3296/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ= github.com/pingcap/tidb-tools v5.0.3+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= @@ -720,6 +726,7 @@ github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114 h1:Pm6R878vxWWW github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0= @@ -790,8 +797,9 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ= github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= -github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210913094925-a8fa8acd44e7 h1:ZK5tbTW6v4dr3OhLqqldgzIlGmZso6KrGjFh7T590VA= github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210913094925-a8fa8acd44e7/go.mod h1:KwtZXt0JD+bP9bWW2ka0ir3Wp3oTEfZUTh22bs2sI4o= +github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210918071319-6ca8098ae0fe h1:nOeupIzlqbMP/UswMxfqvz2YftFa2x+N13QN0QpyiZs= +github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210918071319-6ca8098ae0fe/go.mod h1:KwtZXt0JD+bP9bWW2ka0ir3Wp3oTEfZUTh22bs2sI4o= github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d/go.mod h1:Jw9KG11C/23Rr7DW4XWQ7H5xOgGZo6DFL1OKAF4+Igw= github.com/tikv/pd v1.1.0-beta.0.20210818082359-acba1da0018d h1:AFm1Dzw+QRUevWRfrFp45CPPkuK/zdSWcfxI10z+WVE= github.com/tikv/pd v1.1.0-beta.0.20210818082359-acba1da0018d/go.mod h1:rammPjeZgpvfrQRPkijcx8tlxF1XM5+m6kRXrkDzCAA= @@ -804,7 +812,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM= github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= @@ -846,6 +853,7 @@ github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457/go.mod h1:p github.com/xitongsys/parquet-go v1.6.0 h1:j6YrTVZdQx5yywJLIOklZcKVsCoSD1tqOVRXyTBFSjs= github.com/xitongsys/parquet-go v1.6.0/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8= github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= +github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= @@ -935,6 +943,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= diff --git a/loader/checkpoint.go b/loader/checkpoint.go index 1becbcf6c4..292afc212b 100644 --- a/loader/checkpoint.go +++ b/loader/checkpoint.go @@ -351,7 +351,7 @@ func (cp *RemoteCheckPoint) Init(tctx *tcontext.Context, filename string, endPos cp.connMutex.Unlock() if err != nil { if isErrDupEntry(err) { - cp.logger.Error("checkpoint record already exists, skip it.", zap.String("id", cp.id), zap.String("filename", filename)) + cp.logger.Warn("checkpoint record already exists, skip it.", zap.String("id", cp.id), zap.String("filename", filename)) return nil } return terror.WithScope(terror.Annotate(err, "initialize checkpoint"), terror.ScopeDownstream) diff --git a/loader/lightning.go b/loader/lightning.go new file mode 100644 index 0000000000..5324134528 --- /dev/null +++ b/loader/lightning.go @@ -0,0 +1,319 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package loader + +import ( + "context" + "github.com/pingcap/dm/pkg/binlog" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/pingcap/failpoint" + + "github.com/pingcap/dm/pkg/conn" + + "go.etcd.io/etcd/clientv3" + + "github.com/pingcap/dm/dm/config" + "github.com/pingcap/dm/dm/pb" + "github.com/pingcap/dm/dm/unit" + tcontext "github.com/pingcap/dm/pkg/context" + "github.com/pingcap/dm/pkg/ha" + "github.com/pingcap/dm/pkg/log" + "github.com/pingcap/dm/pkg/utils" + + "github.com/pingcap/tidb/br/pkg/lightning" + lcfg "github.com/pingcap/tidb/br/pkg/lightning/config" + "go.uber.org/atomic" + "go.uber.org/zap" +) + +const ( + lightningCheckpointFile = "lightning.checkpoint.0.sql" + lightningCheckpointDB = "lightning" + lightningCheckpointTable = "checkpoint" +) + +// LightningLoader can load your mydumper data into TiDB database. +type LightningLoader struct { + sync.RWMutex + + cfg *config.SubTaskConfig + cli *clientv3.Client + checkPoint CheckPoint + workerName string + logger log.Logger + core *lightning.Lightning + toDB *conn.BaseDB + toDBConns []*DBConn + lightningConfig *lcfg.GlobalConfig + + finish atomic.Bool + closed atomic.Bool + cancel context.CancelFunc // for per task context, which maybe different from lightning context +} + +// NewLightningLoader creates a new Loader importing data with lightning. +func NewLightningLoader(cfg *config.SubTaskConfig, cli *clientv3.Client, workerName string) *LightningLoader { + lightningCfg := makeGlobalConfig(cfg) + core := lightning.New(lightningCfg) + loader := &LightningLoader{ + cfg: cfg, + cli: cli, + core: core, + lightningConfig: lightningCfg, + logger: log.With(zap.String("task", cfg.Name), zap.String("unit", "lightning-load")), + workerName: workerName, + } + return loader +} + +func makeGlobalConfig(cfg *config.SubTaskConfig) *lcfg.GlobalConfig { + lightningCfg := lcfg.NewGlobalConfig() + if cfg.To.Security != nil { + lightningCfg.Security.CAPath = cfg.To.Security.SSLCA + lightningCfg.Security.CertPath = cfg.To.Security.SSLCert + lightningCfg.Security.KeyPath = cfg.To.Security.SSLKey + } + lightningCfg.TiDB.Host = cfg.To.Host + lightningCfg.TiDB.Psw = cfg.To.Password + lightningCfg.TiDB.User = cfg.To.User + lightningCfg.TiDB.Port = cfg.To.Port + lightningCfg.TiDB.StatusPort = cfg.TiDB.StatusPort + lightningCfg.TiDB.PdAddr = cfg.TiDB.PdAddr + lightningCfg.TikvImporter.Backend = cfg.TiDB.Backend + lightningCfg.PostRestore.Checksum = lcfg.OpLevelOff + if cfg.TiDB.Backend == lcfg.BackendLocal { + lightningCfg.TikvImporter.SortedKVDir = cfg.Dir + } + lightningCfg.Mydumper.SourceDir = cfg.Dir + return lightningCfg +} + +// Type implements Unit.Type. +func (l *LightningLoader) Type() pb.UnitType { + return pb.UnitType_Load +} + +// Init initializes loader for a load task, but not start Process. +// if fail, it should not call l.Close. +func (l *LightningLoader) Init(ctx context.Context) (err error) { + tctx := tcontext.NewContext(ctx, l.logger) + checkpoint, err := newRemoteCheckPoint(tctx, l.cfg, l.checkpointID()) + failpoint.Inject("ignoreLoadCheckpointErr", func(_ failpoint.Value) { + l.logger.Info("", zap.String("failpoint", "ignoreLoadCheckpointErr")) + err = nil + }) + l.checkPoint = checkpoint + l.toDB, l.toDBConns, err = createConns(tctx, l.cfg, 1) + return err +} + +func (l *LightningLoader) restore(ctx context.Context) error { + tctx := tcontext.NewContext(ctx, l.logger) + if err := l.putLoadTask(); err != nil { + return err + } + if err := l.checkPoint.Init(tctx, lightningCheckpointFile, 1); err != nil { + return err + } + if err := l.checkPoint.Load(tctx); err != nil { + return err + } + db2Tables := make(map[string]Tables2DataFiles) + tables := make(Tables2DataFiles) + files := make(DataFiles, 0, 1) + files = append(files, lightningCheckpointFile) + tables[lightningCheckpointTable] = files + db2Tables[lightningCheckpointDB] = tables + var err error + if err = l.checkPoint.CalcProgress(db2Tables); err != nil { + return err + } + if !l.checkPoint.IsTableFinished(lightningCheckpointDB, lightningCheckpointTable) { + cfg := lcfg.NewConfig() + if err = cfg.LoadFromGlobal(l.lightningConfig); err != nil { + return err + } + cfg.Routes = l.cfg.RouteRules + if err = cfg.Adjust(ctx); err != nil { + return err + } + taskCtx, cancel := context.WithCancel(ctx) + l.cancel = cancel + err = l.core.RunOnce(taskCtx, cfg, nil) + if err == nil { + l.finish.Store(true) + offsetSQL := l.checkPoint.GenSQL(lightningCheckpointFile, 1) + err = l.toDBConns[0].executeSQL(tctx, []string{offsetSQL}) + _ = l.checkPoint.UpdateOffset(lightningCheckpointFile, 1) + } + } else { + l.finish.Store(true) + } + if l.cfg.Mode == config.ModeFull { + _ = l.delLoadTask() + } + if l.cfg.CleanDumpFile { + l.cleanDumpFiles() + } + return err +} + +// Process implements Unit.Process. +func (l *LightningLoader) Process(ctx context.Context, pr chan pb.ProcessResult) { + l.logger.Info("lightning load start") + errs := make([]*pb.ProcessError, 0, 1) + if err := l.restore(ctx); err != nil { + errs = append(errs, unit.NewProcessError(err)) + } + l.logger.Info("lightning load end") + pr <- pb.ProcessResult{ + IsCanceled: false, + Errors: errs, + } +} + +func (l *LightningLoader) isClosed() bool { + return l.closed.Load() +} + +// IsFreshTask implements Unit.IsFreshTask. +func (l *LightningLoader) IsFreshTask(ctx context.Context) (bool, error) { + count, err := l.checkPoint.Count(tcontext.NewContext(ctx, l.logger)) + return count == 0, err +} + +// Close does graceful shutdown. +func (l *LightningLoader) Close() { + l.Pause() + l.closed.Store(true) +} + +// Pause pauses the process, and it can be resumed later +// should cancel context from external. +func (l *LightningLoader) Pause() { + l.Lock() + defer l.Unlock() + if l.isClosed() { + l.logger.Warn("try to pause, but already closed") + return + } + if l.cancel != nil { + l.cancel() + } + l.core.Stop() +} + +// Resume resumes the paused process. +func (l *LightningLoader) Resume(ctx context.Context, pr chan pb.ProcessResult) { + if l.isClosed() { + l.logger.Warn("try to resume, but already closed") + return + } + l.core = lightning.New(l.lightningConfig) + // continue the processing + l.Process(ctx, pr) +} + +// Update implements Unit.Update +// now, only support to update config for routes, filters, column-mappings, block-allow-list +// now no config diff implemented, so simply re-init use new config +// no binlog filter for loader need to update. +func (l *LightningLoader) Update(cfg *config.SubTaskConfig) error { + // update l.cfg + l.cfg.BAList = cfg.BAList + l.cfg.RouteRules = cfg.RouteRules + l.cfg.ColumnMappingRules = cfg.ColumnMappingRules + return nil +} + +// Status returns the unit's current status. +func (l *LightningLoader) Status(_ *binlog.SourceStatus) interface{} { + finished, total := l.core.Status() + progress := percent(finished, total, l.finish.Load()) + s := &pb.LoadStatus{ + FinishedBytes: finished, + TotalBytes: total, + Progress: progress, + MetaBinlog: "0", + MetaBinlogGTID: "0", + } + return s +} + +// checkpointID returns ID which used for checkpoint table. +func (l *LightningLoader) checkpointID() string { + if len(l.cfg.SourceID) > 0 { + return l.cfg.SourceID + } + dir, err := filepath.Abs(l.cfg.Dir) + if err != nil { + l.logger.Warn("get abs dir", zap.String("directory", l.cfg.Dir), log.ShortError(err)) + return l.cfg.Dir + } + return shortSha1(dir) +} + +// cleanDumpFiles is called when finish restoring data, to clean useless files. +func (l *LightningLoader) cleanDumpFiles() { + l.logger.Info("clean dump files") + if l.cfg.Mode == config.ModeFull { + // in full-mode all files won't be need in the future + if err := os.RemoveAll(l.cfg.Dir); err != nil { + l.logger.Warn("error when remove loaded dump folder", zap.String("data folder", l.cfg.Dir), zap.Error(err)) + } + } else { + // leave metadata file and table structure files, only delete data files + files, err := utils.CollectDirFiles(l.cfg.Dir) + if err != nil { + l.logger.Warn("fail to collect files", zap.String("data folder", l.cfg.Dir), zap.Error(err)) + } + var lastErr error + for f := range files { + if strings.HasSuffix(f, ".sql") { + // TODO: table structure files are not used now, but we plan to used them in future so not delete them + if strings.HasSuffix(f, "-schema-create.sql") || strings.HasSuffix(f, "-schema.sql") { + continue + } + lastErr = os.Remove(filepath.Join(l.cfg.Dir, f)) + } + } + if lastErr != nil { + l.logger.Warn("show last error when remove loaded dump sql files", zap.String("data folder", l.cfg.Dir), zap.Error(lastErr)) + } + } +} + +// putLoadTask is called when start restoring data, to put load worker in etcd. +func (l *LightningLoader) putLoadTask() error { + _, err := ha.PutLoadTask(l.cli, l.cfg.Name, l.cfg.SourceID, l.workerName) + if err != nil { + return err + } + l.logger.Info("put load worker in etcd", zap.String("task", l.cfg.Name), zap.String("source", l.cfg.SourceID), zap.String("worker", l.workerName)) + return nil +} + +// delLoadTask is called when finish restoring data, to delete load worker in etcd. +func (l *LightningLoader) delLoadTask() error { + _, _, err := ha.DelLoadTask(l.cli, l.cfg.Name, l.cfg.SourceID) + if err != nil { + return err + } + l.logger.Info("delete load worker in etcd for full mode", zap.String("task", l.cfg.Name), zap.String("source", l.cfg.SourceID), zap.String("worker", l.workerName)) + return nil +} diff --git a/pkg/schema/tracker.go b/pkg/schema/tracker.go index ee96c3de71..6a448a5555 100644 --- a/pkg/schema/tracker.go +++ b/pkg/schema/tracker.go @@ -300,7 +300,7 @@ func (tr *Tracker) CreateSchemaIfNotExists(db string) error { if tr.dom.InfoSchema().SchemaExists(dbName) { return nil } - return tr.dom.DDL().CreateSchema(tr.se, dbName, nil) + return tr.dom.DDL().CreateSchema(tr.se, dbName, nil,nil) } // cloneTableInfo creates a clone of the TableInfo. diff --git a/pkg/terror/error_list.go b/pkg/terror/error_list.go index 46a4715b41..a8372698da 100644 --- a/pkg/terror/error_list.go +++ b/pkg/terror/error_list.go @@ -358,6 +358,7 @@ const ( codeLoadUnitGenBAList codeLoadTaskWorkerNotMatch codeLoadCheckPointNotMatch + codeLoadBackendNotMatch ) // Sync unit error code. @@ -981,6 +982,7 @@ var ( ErrLoadUnitGenBAList = New(codeLoadUnitGenBAList, ClassLoadUnit, ScopeInternal, LevelHigh, "generate block allow list", "Please check the `block-allow-list` config in task configuration file.") ErrLoadTaskWorkerNotMatch = New(codeLoadTaskWorkerNotMatch, ClassFunctional, ScopeInternal, LevelHigh, "different worker in load stage, previous worker: %s, current worker: %s", "Please check if the previous worker is online.") ErrLoadTaskCheckPointNotMatch = New(codeLoadCheckPointNotMatch, ClassFunctional, ScopeInternal, LevelHigh, "inconsistent checkpoints between loader and target database", "If you want to redo the whole task, please check that you have not forgotten to add -remove-meta flag for start-task command.") + ErrLoadBackendNotSupport = New(codeLoadBackendNotMatch, ClassFunctional, ScopeInternal, LevelHigh, "DM do not support backend %s ", "If you do not understand the configure `tidb.backend` you can just delete it.") // Sync unit error. ErrSyncerUnitPanic = New(codeSyncerUnitPanic, ClassSyncUnit, ScopeInternal, LevelHigh, "panic error: %v", "") From df036b6c742bdebf49fc3b8271d6410e38f48200 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Fri, 24 Sep 2021 13:35:31 +0800 Subject: [PATCH 02/23] fix bug Signed-off-by: Little-Wallace --- pkg/schema/tracker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/schema/tracker.go b/pkg/schema/tracker.go index 6a448a5555..95193d9a2d 100644 --- a/pkg/schema/tracker.go +++ b/pkg/schema/tracker.go @@ -300,7 +300,7 @@ func (tr *Tracker) CreateSchemaIfNotExists(db string) error { if tr.dom.InfoSchema().SchemaExists(dbName) { return nil } - return tr.dom.DDL().CreateSchema(tr.se, dbName, nil,nil) + return tr.dom.DDL().CreateSchema(tr.se, dbName, nil,nil, nil) } // cloneTableInfo creates a clone of the TableInfo. From 44474f58de795632421ca9a8d4c714424f07d0b3 Mon Sep 17 00:00:00 2001 From: Wallace Date: Fri, 24 Sep 2021 13:34:44 +0800 Subject: [PATCH 03/23] Update dm/config/subtask.go Co-authored-by: lance6716 Signed-off-by: Little-Wallace --- dm/config/subtask.go | 5 ++--- loader/lightning.go | 12 ++++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/dm/config/subtask.go b/dm/config/subtask.go index 013693eaab..822cb1c4cf 100644 --- a/dm/config/subtask.go +++ b/dm/config/subtask.go @@ -23,13 +23,12 @@ import ( "strconv" "strings" - lcfg "github.com/pingcap/tidb/br/pkg/lightning/config" - "github.com/BurntSushi/toml" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" "github.com/pingcap/tidb-tools/pkg/column-mapping" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" + lcfg "github.com/pingcap/tidb/br/pkg/lightning/config" "go.uber.org/zap" "github.com/pingcap/dm/pkg/dumpling" @@ -186,7 +185,7 @@ func GetDBConfigFromEnv() DBConfig { } } -// TiDBExtraConfig is the extra DB configuration only for TiDb. +// TiDBExtraConfig is the extra DB configuration only for TiDB. type TiDBExtraConfig struct { StatusPort int `toml:"status-port" json:"status-port" yaml:"status-port"` PdAddr string `toml:"pd-addr" json:"pd-addr" yaml:"pd-addr"` diff --git a/loader/lightning.go b/loader/lightning.go index 5324134528..7c2052ab21 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -152,8 +152,10 @@ func (l *LightningLoader) restore(ctx context.Context) error { if err = cfg.Adjust(ctx); err != nil { return err } + l.Lock() taskCtx, cancel := context.WithCancel(ctx) l.cancel = cancel + l.Unlock() err = l.core.RunOnce(taskCtx, cfg, nil) if err == nil { l.finish.Store(true) @@ -177,12 +179,18 @@ func (l *LightningLoader) restore(ctx context.Context) error { func (l *LightningLoader) Process(ctx context.Context, pr chan pb.ProcessResult) { l.logger.Info("lightning load start") errs := make([]*pb.ProcessError, 0, 1) - if err := l.restore(ctx); err != nil { + if err := l.restore(ctx); err != nil && !utils.IsContextCanceledError(err) { errs = append(errs, unit.NewProcessError(err)) } + isCanceled := false + select { + case <-ctx.Done(): + isCanceled = true + default: + } l.logger.Info("lightning load end") pr <- pb.ProcessResult{ - IsCanceled: false, + IsCanceled: isCanceled, Errors: errs, } } From 789997c0b1d85b96730295e95d1b446a4790f3f0 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Fri, 24 Sep 2021 14:41:15 +0800 Subject: [PATCH 04/23] fix repeat Signed-off-by: Little-Wallace --- loader/lightning.go | 63 +++++---------------------------------------- loader/loader.go | 36 +++++++++++++------------- 2 files changed, 24 insertions(+), 75 deletions(-) diff --git a/loader/lightning.go b/loader/lightning.go index 7c2052ab21..d8b751fd99 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -16,9 +16,7 @@ package loader import ( "context" "github.com/pingcap/dm/pkg/binlog" - "os" "path/filepath" - "strings" "sync" "github.com/pingcap/failpoint" @@ -31,7 +29,6 @@ import ( "github.com/pingcap/dm/dm/pb" "github.com/pingcap/dm/dm/unit" tcontext "github.com/pingcap/dm/pkg/context" - "github.com/pingcap/dm/pkg/ha" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/utils" @@ -124,7 +121,7 @@ func (l *LightningLoader) Init(ctx context.Context) (err error) { func (l *LightningLoader) restore(ctx context.Context) error { tctx := tcontext.NewContext(ctx, l.logger) - if err := l.putLoadTask(); err != nil { + if err := putLoadTask(l.cli, l.cfg, l.workerName); err != nil { return err } if err := l.checkPoint.Init(tctx, lightningCheckpointFile, 1); err != nil { @@ -167,10 +164,12 @@ func (l *LightningLoader) restore(ctx context.Context) error { l.finish.Store(true) } if l.cfg.Mode == config.ModeFull { - _ = l.delLoadTask() + _ = delLoadTask(l.cli, l.cfg, l.workerName) } - if l.cfg.CleanDumpFile { - l.cleanDumpFiles() + if l.finish.Load() { + if l.cfg.CleanDumpFile { + cleanDumpFiles(l.cfg) + } } return err } @@ -275,53 +274,3 @@ func (l *LightningLoader) checkpointID() string { } return shortSha1(dir) } - -// cleanDumpFiles is called when finish restoring data, to clean useless files. -func (l *LightningLoader) cleanDumpFiles() { - l.logger.Info("clean dump files") - if l.cfg.Mode == config.ModeFull { - // in full-mode all files won't be need in the future - if err := os.RemoveAll(l.cfg.Dir); err != nil { - l.logger.Warn("error when remove loaded dump folder", zap.String("data folder", l.cfg.Dir), zap.Error(err)) - } - } else { - // leave metadata file and table structure files, only delete data files - files, err := utils.CollectDirFiles(l.cfg.Dir) - if err != nil { - l.logger.Warn("fail to collect files", zap.String("data folder", l.cfg.Dir), zap.Error(err)) - } - var lastErr error - for f := range files { - if strings.HasSuffix(f, ".sql") { - // TODO: table structure files are not used now, but we plan to used them in future so not delete them - if strings.HasSuffix(f, "-schema-create.sql") || strings.HasSuffix(f, "-schema.sql") { - continue - } - lastErr = os.Remove(filepath.Join(l.cfg.Dir, f)) - } - } - if lastErr != nil { - l.logger.Warn("show last error when remove loaded dump sql files", zap.String("data folder", l.cfg.Dir), zap.Error(lastErr)) - } - } -} - -// putLoadTask is called when start restoring data, to put load worker in etcd. -func (l *LightningLoader) putLoadTask() error { - _, err := ha.PutLoadTask(l.cli, l.cfg.Name, l.cfg.SourceID, l.workerName) - if err != nil { - return err - } - l.logger.Info("put load worker in etcd", zap.String("task", l.cfg.Name), zap.String("source", l.cfg.SourceID), zap.String("worker", l.workerName)) - return nil -} - -// delLoadTask is called when finish restoring data, to delete load worker in etcd. -func (l *LightningLoader) delLoadTask() error { - _, _, err := ha.DelLoadTask(l.cli, l.cfg.Name, l.cfg.SourceID) - if err != nil { - return err - } - l.logger.Info("delete load worker in etcd for full mode", zap.String("task", l.cfg.Name), zap.String("source", l.cfg.SourceID), zap.String("worker", l.workerName)) - return nil -} diff --git a/loader/loader.go b/loader/loader.go index 176c978b8b..4a77e10aec 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -693,7 +693,7 @@ func (l *Loader) IsFreshTask(ctx context.Context) (bool, error) { // Restore begins the restore process. func (l *Loader) Restore(ctx context.Context) error { - if err := l.putLoadTask(); err != nil { + if err := putLoadTask(l.cli, l.cfg, l.workerName); err != nil { return err } @@ -743,12 +743,12 @@ func (l *Loader) Restore(ctx context.Context) error { l.logger.Info("all data files have been finished", zap.Duration("cost time", time.Since(begin))) if l.checkPoint.AllFinished() { if l.cfg.Mode == config.ModeFull { - if err = l.delLoadTask(); err != nil { + if err = delLoadTask(l.cli, l.cfg, l.workerName); err != nil { return err } } if l.cfg.CleanDumpFile { - l.cleanDumpFiles() + cleanDumpFiles(l.cfg) } } } else if errors.Cause(err) != context.Canceled { @@ -1540,18 +1540,18 @@ func (l *Loader) getMydumpMetadata() error { } // cleanDumpFiles is called when finish restoring data, to clean useless files. -func (l *Loader) cleanDumpFiles() { - l.logger.Info("clean dump files") - if l.cfg.Mode == config.ModeFull { +func cleanDumpFiles(cfg *config.SubTaskConfig) { + log.L().Info("clean dump files") + if cfg.Mode == config.ModeFull { // in full-mode all files won't be need in the future - if err := os.RemoveAll(l.cfg.Dir); err != nil { - l.logger.Warn("error when remove loaded dump folder", zap.String("data folder", l.cfg.Dir), zap.Error(err)) + if err := os.RemoveAll(cfg.Dir); err != nil { + log.L().Warn("error when remove loaded dump folder", zap.String("data folder", cfg.Dir), zap.Error(err)) } } else { // leave metadata file and table structure files, only delete data files - files, err := utils.CollectDirFiles(l.cfg.Dir) + files, err := utils.CollectDirFiles(cfg.Dir) if err != nil { - l.logger.Warn("fail to collect files", zap.String("data folder", l.cfg.Dir), zap.Error(err)) + log.L().Warn("fail to collect files", zap.String("data folder", cfg.Dir), zap.Error(err)) } var lastErr error for f := range files { @@ -1560,31 +1560,31 @@ func (l *Loader) cleanDumpFiles() { if strings.HasSuffix(f, "-schema-create.sql") || strings.HasSuffix(f, "-schema.sql") { continue } - lastErr = os.Remove(filepath.Join(l.cfg.Dir, f)) + lastErr = os.Remove(filepath.Join(cfg.Dir, f)) } } if lastErr != nil { - l.logger.Warn("show last error when remove loaded dump sql files", zap.String("data folder", l.cfg.Dir), zap.Error(lastErr)) + log.L().Warn("show last error when remove loaded dump sql files", zap.String("data folder", cfg.Dir), zap.Error(lastErr)) } } } // putLoadTask is called when start restoring data, to put load worker in etcd. -func (l *Loader) putLoadTask() error { - _, err := ha.PutLoadTask(l.cli, l.cfg.Name, l.cfg.SourceID, l.workerName) +func putLoadTask(cli *clientv3.Client, cfg *config.SubTaskConfig, workerName string) error { + _, err := ha.PutLoadTask(cli, cfg.Name, cfg.SourceID, workerName) if err != nil { return err } - l.logger.Info("put load worker in etcd", zap.String("task", l.cfg.Name), zap.String("source", l.cfg.SourceID), zap.String("worker", l.workerName)) + log.L().Info("put load worker in etcd", zap.String("task", cfg.Name), zap.String("source", cfg.SourceID), zap.String("worker", workerName)) return nil } // delLoadTask is called when finish restoring data, to delete load worker in etcd. -func (l *Loader) delLoadTask() error { - _, _, err := ha.DelLoadTask(l.cli, l.cfg.Name, l.cfg.SourceID) +func delLoadTask(cli *clientv3.Client, cfg *config.SubTaskConfig, workerName string) error { + _, _, err := ha.DelLoadTask(cli, cfg.Name, cfg.SourceID) if err != nil { return err } - l.logger.Info("delete load worker in etcd for full mode", zap.String("task", l.cfg.Name), zap.String("source", l.cfg.SourceID), zap.String("worker", l.workerName)) + log.L().Info("delete load worker in etcd for full mode", zap.String("task", cfg.Name), zap.String("source", cfg.SourceID), zap.String("worker", workerName)) return nil } From 490247cee7b6e36bc30f9aea9a07649a88558824 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Fri, 24 Sep 2021 14:48:10 +0800 Subject: [PATCH 05/23] address comment Signed-off-by: Little-Wallace --- dm/config/task.go | 3 +++ dm/master/server.go | 5 ----- dm/worker/subtask.go | 2 +- loader/lightning.go | 15 ++++++--------- pkg/schema/tracker.go | 2 +- 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/dm/config/task.go b/dm/config/task.go index c4cc229f80..17982402d0 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -781,6 +781,9 @@ func (c *TaskConfig) SubTaskConfigs(sources map[string]DBConfig) ([]*SubTaskConf return nil, terror.Annotatef(err, "source %s", inst.SourceID) } + if c.TiDB != nil { + cfg.TiDB = *c.TiDB + } cfgs[i] = cfg } return cfgs, nil diff --git a/dm/master/server.go b/dm/master/server.go index 172359a924..11c5d48ecc 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -1373,11 +1373,6 @@ func (s *Server) generateSubTask(ctx context.Context, task string, errCnt, warnC sourceCfgs := s.getSourceConfigs(cfg.MySQLInstances) stCfgs, err := cfg.SubTaskConfigs(sourceCfgs) - if cfg.TiDB != nil { - for _, st := range stCfgs { - st.TiDB = *cfg.TiDB - } - } if err != nil { return nil, nil, terror.WithClass(err, terror.ClassDMMaster) } diff --git a/dm/worker/subtask.go b/dm/worker/subtask.go index cedda0f7ec..863c42f1dc 100644 --- a/dm/worker/subtask.go +++ b/dm/worker/subtask.go @@ -62,7 +62,7 @@ func createRealUnits(cfg *config.SubTaskConfig, etcdClient *clientv3.Client, wor if cfg.TiDB.Backend == "" { us = append(us, loader.NewLoader(cfg, etcdClient, workerName)) } else { - us = append(us, loader.NewLightningLoader(cfg, etcdClient, workerName)) + us = append(us, loader.NewLightning(cfg, etcdClient, workerName)) } us = append(us, syncer.NewSyncer(cfg, etcdClient)) case config.ModeFull: diff --git a/loader/lightning.go b/loader/lightning.go index d8b751fd99..4853d8d3d7 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -15,25 +15,22 @@ package loader import ( "context" - "github.com/pingcap/dm/pkg/binlog" "path/filepath" "sync" - "github.com/pingcap/failpoint" - - "github.com/pingcap/dm/pkg/conn" - - "go.etcd.io/etcd/clientv3" - "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/dm/pb" "github.com/pingcap/dm/dm/unit" + "github.com/pingcap/dm/pkg/binlog" + "github.com/pingcap/dm/pkg/conn" tcontext "github.com/pingcap/dm/pkg/context" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/utils" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/br/pkg/lightning" lcfg "github.com/pingcap/tidb/br/pkg/lightning/config" + "go.etcd.io/etcd/clientv3" "go.uber.org/atomic" "go.uber.org/zap" ) @@ -63,8 +60,8 @@ type LightningLoader struct { cancel context.CancelFunc // for per task context, which maybe different from lightning context } -// NewLightningLoader creates a new Loader importing data with lightning. -func NewLightningLoader(cfg *config.SubTaskConfig, cli *clientv3.Client, workerName string) *LightningLoader { +// NewLightning creates a new Loader importing data with lightning. +func NewLightning(cfg *config.SubTaskConfig, cli *clientv3.Client, workerName string) *LightningLoader { lightningCfg := makeGlobalConfig(cfg) core := lightning.New(lightningCfg) loader := &LightningLoader{ diff --git a/pkg/schema/tracker.go b/pkg/schema/tracker.go index 95193d9a2d..50ebc06a1b 100644 --- a/pkg/schema/tracker.go +++ b/pkg/schema/tracker.go @@ -300,7 +300,7 @@ func (tr *Tracker) CreateSchemaIfNotExists(db string) error { if tr.dom.InfoSchema().SchemaExists(dbName) { return nil } - return tr.dom.DDL().CreateSchema(tr.se, dbName, nil,nil, nil) + return tr.dom.DDL().CreateSchema(tr.se, dbName, nil, nil, nil) } // cloneTableInfo creates a clone of the TableInfo. From a8363893effda71515caa86fb658cfe1fcf36b77 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Fri, 24 Sep 2021 18:16:59 +0800 Subject: [PATCH 06/23] add config Signed-off-by: Little-Wallace --- _utils/terror_gen/errors_release.txt | 1 + errors.toml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/_utils/terror_gen/errors_release.txt b/_utils/terror_gen/errors_release.txt index 2c3e3e7a8c..8f0b68e5b2 100644 --- a/_utils/terror_gen/errors_release.txt +++ b/_utils/terror_gen/errors_release.txt @@ -265,6 +265,7 @@ ErrLoadUnitDuplicateTableFile,[code=34015:class=load-unit:scope=internal:level=h ErrLoadUnitGenBAList,[code=34016:class=load-unit:scope=internal:level=high], "Message: generate block allow list, Workaround: Please check the `block-allow-list` config in task configuration file." ErrLoadTaskWorkerNotMatch,[code=34017:class=functional:scope=internal:level=high], "Message: different worker in load stage, previous worker: %s, current worker: %s, Workaround: Please check if the previous worker is online." ErrLoadTaskCheckPointNotMatch,[code=34018:class=functional:scope=internal:level=high], "Message: inconsistent checkpoints between loader and target database, Workaround: If you want to redo the whole task, please check that you have not forgotten to add -remove-meta flag for start-task command." +ErrLoadBackendNotSupport,[code=34019:class=functional:scope=internal:level=high], "Message: DM do not support backend %s , Workaround: If you do not understand the configure `tidb.backend` you can just delete it." ErrSyncerUnitPanic,[code=36001:class=sync-unit:scope=internal:level=high], "Message: panic error: %v" ErrSyncUnitInvalidTableName,[code=36002:class=sync-unit:scope=internal:level=high], "Message: extract table name for DML error: %s" ErrSyncUnitTableNameQuery,[code=36003:class=sync-unit:scope=internal:level=high], "Message: table name parse error: %s" diff --git a/errors.toml b/errors.toml index a80db8e683..35ea8ffdc1 100644 --- a/errors.toml +++ b/errors.toml @@ -1600,6 +1600,12 @@ description = "" workaround = "If you want to redo the whole task, please check that you have not forgotten to add -remove-meta flag for start-task command." tags = ["internal", "high"] +[error.DM-functional-34019] +message = "DM do not support backend %s " +description = "" +workaround = "If you do not understand the configure `tidb.backend` you can just delete it." +tags = ["internal", "high"] + [error.DM-sync-unit-36001] message = "panic error: %v" description = "" From c2edae16f47fdcfd44c298fd119395d431a25d9b Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Sun, 26 Sep 2021 14:36:09 +0800 Subject: [PATCH 07/23] change test to tidb backend Signed-off-by: Little-Wallace --- loader/lightning.go | 15 +++++++++++++++ tests/all_mode/conf/dm-task.yaml | 3 +++ 2 files changed, 18 insertions(+) diff --git a/loader/lightning.go b/loader/lightning.go index 4853d8d3d7..d3f35ca1ac 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -15,6 +15,9 @@ package loader import ( "context" + "github.com/docker/go-units" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/br/pkg/lightning/common" "path/filepath" "sync" @@ -143,6 +146,18 @@ func (l *LightningLoader) restore(ctx context.Context) error { return err } cfg.Routes = l.cfg.RouteRules + cfg.Checkpoint.Schema = "tidb_lightning_checkpoint_" + l.cfg.Name + cfg.Checkpoint.Driver = lcfg.CheckpointDriverMySQL + param := common.MySQLConnectParam{ + Host: cfg.TiDB.Host, + Port: cfg.TiDB.Port, + User: cfg.TiDB.User, + Password: cfg.TiDB.Psw, + SQLMode: mysql.DefaultSQLMode, + MaxAllowedPacket: 64 * units.MiB, + TLS: cfg.TiDB.TLS, + } + cfg.Checkpoint.DSN = param.ToDSN() if err = cfg.Adjust(ctx); err != nil { return err } diff --git a/tests/all_mode/conf/dm-task.yaml b/tests/all_mode/conf/dm-task.yaml index ddc192017f..2acec7b821 100644 --- a/tests/all_mode/conf/dm-task.yaml +++ b/tests/all_mode/conf/dm-task.yaml @@ -44,6 +44,9 @@ filters: # filter rules, mysql instance can ref rules in it events: ["truncate table", "drop table"] # ignore truncate/drop table ddl action: Ignore +tidb: + backend: "tidb" + mydumpers: global: threads: 4 From b9500883f1929da87029453044250aec4f4256a0 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Sun, 26 Sep 2021 16:48:16 +0800 Subject: [PATCH 08/23] add some comment Signed-off-by: Little-Wallace --- dm/master/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/dm/master/server.go b/dm/master/server.go index 11c5d48ecc..10669ef732 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -465,6 +465,7 @@ func (s *Server) StartTask(ctx context.Context, req *pb.StartTaskRequest) (*pb.S ) if req.RemoveMeta { + // TODO: Remove lightning checkpoint and meta. // use same latch for remove-meta and start-task release, err3 = s.scheduler.AcquireSubtaskLatch(cfg.Name) if err3 != nil { From 17e9fcdd212c6616625c7834d255d7c69eb6678d Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Mon, 27 Sep 2021 12:16:14 +0800 Subject: [PATCH 09/23] address comment Signed-off-by: Little-Wallace --- loader/lightning.go | 32 +++++++++++---- loader/loader.go | 95 +++++---------------------------------------- loader/util.go | 89 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 93 deletions(-) diff --git a/loader/lightning.go b/loader/lightning.go index d3f35ca1ac..606be29337 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -15,11 +15,12 @@ package loader import ( "context" + "path/filepath" + "sync" + "github.com/docker/go-units" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/br/pkg/lightning/common" - "path/filepath" - "sync" "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/dm/pb" @@ -58,9 +59,11 @@ type LightningLoader struct { toDBConns []*DBConn lightningConfig *lcfg.GlobalConfig - finish atomic.Bool - closed atomic.Bool - cancel context.CancelFunc // for per task context, which maybe different from lightning context + finish atomic.Bool + closed atomic.Bool + metaBinlog atomic.String + metaBinlogGTID atomic.String + cancel context.CancelFunc // for per task context, which maybe different from lightning context } // NewLightning creates a new Loader importing data with lightning. @@ -190,6 +193,21 @@ func (l *LightningLoader) restore(ctx context.Context) error { func (l *LightningLoader) Process(ctx context.Context, pr chan pb.ProcessResult) { l.logger.Info("lightning load start") errs := make([]*pb.ProcessError, 0, 1) + err, binlog, gtid := getMydumpMetadata(l.cli, l.cfg, l.workerName) + if err != nil { + loaderExitWithErrorCounter.WithLabelValues(l.cfg.Name, l.cfg.SourceID).Inc() + pr <- pb.ProcessResult{ + Errors: []*pb.ProcessError{unit.NewProcessError(err)}, + } + return + } + if binlog != "" { + l.metaBinlog.Store(binlog) + } + if gtid != "" { + l.metaBinlogGTID.Store(gtid) + } + if err := l.restore(ctx); err != nil && !utils.IsContextCanceledError(err) { errs = append(errs, unit.NewProcessError(err)) } @@ -268,8 +286,8 @@ func (l *LightningLoader) Status(_ *binlog.SourceStatus) interface{} { FinishedBytes: finished, TotalBytes: total, Progress: progress, - MetaBinlog: "0", - MetaBinlogGTID: "0", + MetaBinlog: l.metaBinlog.Load(), + MetaBinlogGTID: l.metaBinlogGTID.Load(), } return s } diff --git a/loader/loader.go b/loader/loader.go index 4a77e10aec..baae84d48f 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -19,7 +19,6 @@ import ( "context" "encoding/hex" "io" - "io/ioutil" "os" "path/filepath" "strings" @@ -34,9 +33,7 @@ import ( "github.com/pingcap/dm/dm/unit" "github.com/pingcap/dm/pkg/conn" tcontext "github.com/pingcap/dm/pkg/context" - "github.com/pingcap/dm/pkg/dumpling" fr "github.com/pingcap/dm/pkg/func-rollback" - "github.com/pingcap/dm/pkg/ha" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/terror" "github.com/pingcap/dm/pkg/utils" @@ -573,13 +570,20 @@ func (l *Loader) Process(ctx context.Context, pr chan pb.ProcessResult) { defer cancel() l.newFileJobQueue() - if err := l.getMydumpMetadata(); err != nil { + err, binlog, gtid := getMydumpMetadata(l.cli, l.cfg, l.workerName) + if err != nil { loaderExitWithErrorCounter.WithLabelValues(l.cfg.Name, l.cfg.SourceID).Inc() pr <- pb.ProcessResult{ Errors: []*pb.ProcessError{unit.NewProcessError(err)}, } return } + if binlog != "" { + l.metaBinlog.Store(binlog) + } + if gtid != "" { + l.metaBinlogGTID.Store(gtid) + } l.runFatalChan = make(chan *pb.ProcessError, 2*l.cfg.PoolSize) errs := make([]*pb.ProcessError, 0, 2) @@ -594,7 +598,7 @@ func (l *Loader) Process(ctx context.Context, pr chan pb.ProcessResult) { } }() - err := l.Restore(newCtx) + err = l.Restore(newCtx) close(l.runFatalChan) // Restore returned, all potential fatal sent to l.runFatalChan cancel() // cancel the goroutines created in `Restore`. @@ -1507,84 +1511,3 @@ func (l *Loader) checkpointID() string { } return shortSha1(dir) } - -func (l *Loader) getMydumpMetadata() error { - metafile := filepath.Join(l.cfg.LoaderConfig.Dir, "metadata") - loc, _, err := dumpling.ParseMetaData(metafile, l.cfg.Flavor) - if err != nil { - if os.IsNotExist(err) { - worker, _, err2 := ha.GetLoadTask(l.cli, l.cfg.Name, l.cfg.SourceID) - if err2 != nil { - l.logger.Warn("get load task", log.ShortError(err2)) - } - if worker != "" && worker != l.workerName { - return terror.ErrLoadTaskWorkerNotMatch.Generate(worker, l.workerName) - } - } - if terror.ErrMetadataNoBinlogLoc.Equal(err) { - l.logger.Warn("dumped metadata doesn't have binlog location, it's OK if DM doesn't enter incremental mode") - return nil - } - - toPrint, err2 := ioutil.ReadFile(metafile) - if err2 != nil { - toPrint = []byte(err2.Error()) - } - l.logger.Error("fail to parse dump metadata", log.ShortError(err)) - return terror.ErrParseMydumperMeta.Generate(err, toPrint) - } - - l.metaBinlog.Store(loc.Position.String()) - l.metaBinlogGTID.Store(loc.GTIDSetStr()) - return nil -} - -// cleanDumpFiles is called when finish restoring data, to clean useless files. -func cleanDumpFiles(cfg *config.SubTaskConfig) { - log.L().Info("clean dump files") - if cfg.Mode == config.ModeFull { - // in full-mode all files won't be need in the future - if err := os.RemoveAll(cfg.Dir); err != nil { - log.L().Warn("error when remove loaded dump folder", zap.String("data folder", cfg.Dir), zap.Error(err)) - } - } else { - // leave metadata file and table structure files, only delete data files - files, err := utils.CollectDirFiles(cfg.Dir) - if err != nil { - log.L().Warn("fail to collect files", zap.String("data folder", cfg.Dir), zap.Error(err)) - } - var lastErr error - for f := range files { - if strings.HasSuffix(f, ".sql") { - // TODO: table structure files are not used now, but we plan to used them in future so not delete them - if strings.HasSuffix(f, "-schema-create.sql") || strings.HasSuffix(f, "-schema.sql") { - continue - } - lastErr = os.Remove(filepath.Join(cfg.Dir, f)) - } - } - if lastErr != nil { - log.L().Warn("show last error when remove loaded dump sql files", zap.String("data folder", cfg.Dir), zap.Error(lastErr)) - } - } -} - -// putLoadTask is called when start restoring data, to put load worker in etcd. -func putLoadTask(cli *clientv3.Client, cfg *config.SubTaskConfig, workerName string) error { - _, err := ha.PutLoadTask(cli, cfg.Name, cfg.SourceID, workerName) - if err != nil { - return err - } - log.L().Info("put load worker in etcd", zap.String("task", cfg.Name), zap.String("source", cfg.SourceID), zap.String("worker", workerName)) - return nil -} - -// delLoadTask is called when finish restoring data, to delete load worker in etcd. -func delLoadTask(cli *clientv3.Client, cfg *config.SubTaskConfig, workerName string) error { - _, _, err := ha.DelLoadTask(cli, cfg.Name, cfg.SourceID) - if err != nil { - return err - } - log.L().Info("delete load worker in etcd for full mode", zap.String("task", cfg.Name), zap.String("source", cfg.SourceID), zap.String("worker", workerName)) - return nil -} diff --git a/loader/util.go b/loader/util.go index 309bfa0971..a88af30a0b 100644 --- a/loader/util.go +++ b/loader/util.go @@ -16,11 +16,21 @@ package loader import ( "crypto/sha1" "fmt" + "io/ioutil" "os" "path" + "path/filepath" "strings" + "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" + + "github.com/pingcap/dm/dm/config" + "github.com/pingcap/dm/pkg/dumpling" + "github.com/pingcap/dm/pkg/ha" + "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/terror" + "github.com/pingcap/dm/pkg/utils" ) // SQLReplace works like strings.Replace but only supports one replacement. @@ -102,3 +112,82 @@ func getDBAndTableFromFilename(filename string) (string, string, error) { } return fields[0], fields[1], nil } + +func getMydumpMetadata(cli *clientv3.Client, cfg *config.SubTaskConfig, workerName string) (error, string, string) { + metafile := filepath.Join(cfg.LoaderConfig.Dir, "metadata") + loc, _, err := dumpling.ParseMetaData(metafile, cfg.Flavor) + if err != nil { + if os.IsNotExist(err) { + worker, _, err2 := ha.GetLoadTask(cli, cfg.Name, cfg.SourceID) + if err2 != nil { + log.L().Warn("get load task", log.ShortError(err2)) + } + if worker != "" && worker != workerName { + return terror.ErrLoadTaskWorkerNotMatch.Generate(worker, workerName), "", "" + } + } + if terror.ErrMetadataNoBinlogLoc.Equal(err) { + log.L().Warn("dumped metadata doesn't have binlog location, it's OK if DM doesn't enter incremental mode") + return nil, "", "" + } + + toPrint, err2 := ioutil.ReadFile(metafile) + if err2 != nil { + toPrint = []byte(err2.Error()) + } + log.L().Error("fail to parse dump metadata", log.ShortError(err)) + return terror.ErrParseMydumperMeta.Generate(err, toPrint), "", "" + } + + return nil, loc.Position.String(), loc.GTIDSetStr() +} + +// cleanDumpFiles is called when finish restoring data, to clean useless files. +func cleanDumpFiles(cfg *config.SubTaskConfig) { + log.L().Info("clean dump files") + if cfg.Mode == config.ModeFull { + // in full-mode all files won't be need in the future + if err := os.RemoveAll(cfg.Dir); err != nil { + log.L().Warn("error when remove loaded dump folder", zap.String("data folder", cfg.Dir), zap.Error(err)) + } + } else { + // leave metadata file and table structure files, only delete data files + files, err := utils.CollectDirFiles(cfg.Dir) + if err != nil { + log.L().Warn("fail to collect files", zap.String("data folder", cfg.Dir), zap.Error(err)) + } + var lastErr error + for f := range files { + if strings.HasSuffix(f, ".sql") { + // TODO: table structure files are not used now, but we plan to used them in future so not delete them + if strings.HasSuffix(f, "-schema-create.sql") || strings.HasSuffix(f, "-schema.sql") { + continue + } + lastErr = os.Remove(filepath.Join(cfg.Dir, f)) + } + } + if lastErr != nil { + log.L().Warn("show last error when remove loaded dump sql files", zap.String("data folder", cfg.Dir), zap.Error(lastErr)) + } + } +} + +// putLoadTask is called when start restoring data, to put load worker in etcd. +func putLoadTask(cli *clientv3.Client, cfg *config.SubTaskConfig, workerName string) error { + _, err := ha.PutLoadTask(cli, cfg.Name, cfg.SourceID, workerName) + if err != nil { + return err + } + log.L().Info("put load worker in etcd", zap.String("task", cfg.Name), zap.String("source", cfg.SourceID), zap.String("worker", workerName)) + return nil +} + +// delLoadTask is called when finish restoring data, to delete load worker in etcd. +func delLoadTask(cli *clientv3.Client, cfg *config.SubTaskConfig, workerName string) error { + _, _, err := ha.DelLoadTask(cli, cfg.Name, cfg.SourceID) + if err != nil { + return err + } + log.L().Info("delete load worker in etcd for full mode", zap.String("task", cfg.Name), zap.String("source", cfg.SourceID), zap.String("worker", workerName)) + return nil +} From f9a73a53356a8cfa562dd6c38d65749caeab2414 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Tue, 28 Sep 2021 01:03:22 +0800 Subject: [PATCH 10/23] fix fmt Signed-off-by: Little-Wallace --- loader/lightning.go | 2 +- loader/loader.go | 2 +- loader/util.go | 10 +- tests/all_mode/conf/dm-task.yaml | 3 - tests/dmctl_basic/conf/get_task.yaml | 1 + tests/lightning_mode/conf/diff_config.toml | 56 ++++ tests/lightning_mode/conf/dm-master.toml | 6 + tests/lightning_mode/conf/dm-task.yaml | 63 +++++ tests/lightning_mode/conf/dm-worker1.toml | 2 + tests/lightning_mode/conf/dm-worker2.toml | 2 + tests/lightning_mode/conf/source1.yaml | 18 ++ tests/lightning_mode/conf/source2.yaml | 21 ++ tests/lightning_mode/data/db1.increment.sql | 48 ++++ tests/lightning_mode/data/db1.increment0.sql | 3 + tests/lightning_mode/data/db1.prepare.sql | 20 ++ tests/lightning_mode/data/db2.increment.sql | 5 + tests/lightning_mode/data/db2.increment0.sql | 1 + tests/lightning_mode/data/db2.prepare.sql | 15 ++ tests/lightning_mode/run.sh | 259 +++++++++++++++++++ 19 files changed, 527 insertions(+), 10 deletions(-) create mode 100644 tests/lightning_mode/conf/diff_config.toml create mode 100644 tests/lightning_mode/conf/dm-master.toml create mode 100644 tests/lightning_mode/conf/dm-task.yaml create mode 100644 tests/lightning_mode/conf/dm-worker1.toml create mode 100644 tests/lightning_mode/conf/dm-worker2.toml create mode 100644 tests/lightning_mode/conf/source1.yaml create mode 100644 tests/lightning_mode/conf/source2.yaml create mode 100644 tests/lightning_mode/data/db1.increment.sql create mode 100644 tests/lightning_mode/data/db1.increment0.sql create mode 100644 tests/lightning_mode/data/db1.prepare.sql create mode 100644 tests/lightning_mode/data/db2.increment.sql create mode 100644 tests/lightning_mode/data/db2.increment0.sql create mode 100644 tests/lightning_mode/data/db2.prepare.sql create mode 100755 tests/lightning_mode/run.sh diff --git a/loader/lightning.go b/loader/lightning.go index 606be29337..51dfa22ba8 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -193,7 +193,7 @@ func (l *LightningLoader) restore(ctx context.Context) error { func (l *LightningLoader) Process(ctx context.Context, pr chan pb.ProcessResult) { l.logger.Info("lightning load start") errs := make([]*pb.ProcessError, 0, 1) - err, binlog, gtid := getMydumpMetadata(l.cli, l.cfg, l.workerName) + binlog, gtid, err := getMydumpMetadata(l.cli, l.cfg, l.workerName) if err != nil { loaderExitWithErrorCounter.WithLabelValues(l.cfg.Name, l.cfg.SourceID).Inc() pr <- pb.ProcessResult{ diff --git a/loader/loader.go b/loader/loader.go index baae84d48f..1d9f4dde01 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -570,7 +570,7 @@ func (l *Loader) Process(ctx context.Context, pr chan pb.ProcessResult) { defer cancel() l.newFileJobQueue() - err, binlog, gtid := getMydumpMetadata(l.cli, l.cfg, l.workerName) + binlog, gtid, err := getMydumpMetadata(l.cli, l.cfg, l.workerName) if err != nil { loaderExitWithErrorCounter.WithLabelValues(l.cfg.Name, l.cfg.SourceID).Inc() pr <- pb.ProcessResult{ diff --git a/loader/util.go b/loader/util.go index a88af30a0b..d6396e000a 100644 --- a/loader/util.go +++ b/loader/util.go @@ -113,7 +113,7 @@ func getDBAndTableFromFilename(filename string) (string, string, error) { return fields[0], fields[1], nil } -func getMydumpMetadata(cli *clientv3.Client, cfg *config.SubTaskConfig, workerName string) (error, string, string) { +func getMydumpMetadata(cli *clientv3.Client, cfg *config.SubTaskConfig, workerName string) (string, string, error) { metafile := filepath.Join(cfg.LoaderConfig.Dir, "metadata") loc, _, err := dumpling.ParseMetaData(metafile, cfg.Flavor) if err != nil { @@ -123,12 +123,12 @@ func getMydumpMetadata(cli *clientv3.Client, cfg *config.SubTaskConfig, workerNa log.L().Warn("get load task", log.ShortError(err2)) } if worker != "" && worker != workerName { - return terror.ErrLoadTaskWorkerNotMatch.Generate(worker, workerName), "", "" + return "", "", terror.ErrLoadTaskWorkerNotMatch.Generate(worker, workerName) } } if terror.ErrMetadataNoBinlogLoc.Equal(err) { log.L().Warn("dumped metadata doesn't have binlog location, it's OK if DM doesn't enter incremental mode") - return nil, "", "" + return "", "", nil } toPrint, err2 := ioutil.ReadFile(metafile) @@ -136,10 +136,10 @@ func getMydumpMetadata(cli *clientv3.Client, cfg *config.SubTaskConfig, workerNa toPrint = []byte(err2.Error()) } log.L().Error("fail to parse dump metadata", log.ShortError(err)) - return terror.ErrParseMydumperMeta.Generate(err, toPrint), "", "" + return "", "", terror.ErrParseMydumperMeta.Generate(err, toPrint) } - return nil, loc.Position.String(), loc.GTIDSetStr() + return loc.Position.String(), loc.GTIDSetStr(), nil } // cleanDumpFiles is called when finish restoring data, to clean useless files. diff --git a/tests/all_mode/conf/dm-task.yaml b/tests/all_mode/conf/dm-task.yaml index 2acec7b821..ddc192017f 100644 --- a/tests/all_mode/conf/dm-task.yaml +++ b/tests/all_mode/conf/dm-task.yaml @@ -44,9 +44,6 @@ filters: # filter rules, mysql instance can ref rules in it events: ["truncate table", "drop table"] # ignore truncate/drop table ddl action: Ignore -tidb: - backend: "tidb" - mydumpers: global: threads: 4 diff --git a/tests/dmctl_basic/conf/get_task.yaml b/tests/dmctl_basic/conf/get_task.yaml index c2b79d4a82..bb86061a95 100644 --- a/tests/dmctl_basic/conf/get_task.yaml +++ b/tests/dmctl_basic/conf/get_task.yaml @@ -158,3 +158,4 @@ syncers: clean-dump-file: true ansi-quotes: false remove-meta: false +tidb: null diff --git a/tests/lightning_mode/conf/diff_config.toml b/tests/lightning_mode/conf/diff_config.toml new file mode 100644 index 0000000000..6dfc688aaf --- /dev/null +++ b/tests/lightning_mode/conf/diff_config.toml @@ -0,0 +1,56 @@ +# diff Configuration. + +log-level = "info" + +chunk-size = 1000 + +check-thread-count = 4 + +sample-percent = 100 + +use-checksum = true + +fix-sql-file = "fix.sql" + +# tables need to check. +[[check-tables]] +schema = "all_mode" +tables = ["~t.*"] + +[[table-config]] +schema = "all_mode" +table = "t1" + +[[table-config.source-tables]] +instance-id = "source-1" +schema = "all_mode" +table = "t1" + +[[table-config]] +schema = "all_mode" +table = "t2" + +[[table-config.source-tables]] +instance-id = "source-2" +schema = "all_mode" +table = "t2" + +[[source-db]] +host = "127.0.0.1" +port = 3306 +user = "root" +password = "123456" +instance-id = "source-1" + +[[source-db]] +host = "127.0.0.1" +port = 3307 +user = "root" +password = "123456" +instance-id = "source-2" + +[target-db] +host = "127.0.0.1" +port = 4000 +user = "test" +password = "123456" diff --git a/tests/lightning_mode/conf/dm-master.toml b/tests/lightning_mode/conf/dm-master.toml new file mode 100644 index 0000000000..53a294e7d0 --- /dev/null +++ b/tests/lightning_mode/conf/dm-master.toml @@ -0,0 +1,6 @@ +# Master Configuration. +master-addr = ":8261" +advertise-addr = "127.0.0.1:8261" + +rpc-timeout = "30s" +auto-compaction-retention = "3s" diff --git a/tests/lightning_mode/conf/dm-task.yaml b/tests/lightning_mode/conf/dm-task.yaml new file mode 100644 index 0000000000..ddc192017f --- /dev/null +++ b/tests/lightning_mode/conf/dm-task.yaml @@ -0,0 +1,63 @@ +--- +name: test +task-mode: all +is-sharding: false +meta-schema: "dm_meta" +# enable-heartbeat: true +heartbeat-update-interval: 1 +heartbeat-report-interval: 1 + +target-database: + host: "127.0.0.1" + port: 4000 + user: "root" + password: "" + session: + tidb_skip_utf8_check: 1 + tidb_disable_txn_auto_retry: off + tidb_retry_limit: "10" + +mysql-instances: + - source-id: "mysql-replica-01" + black-white-list: "instance" # compatible with deprecated config + mydumper-config-name: "global" + loader-config-name: "global" + syncer-config-name: "global" + filter-rules: ["user-filter-1"] + + - source-id: "mysql-replica-02" + block-allow-list: "instance" + mydumper-config-name: "global" + loader-config-name: "global" + syncer-config-name: "global" + filter-rules: ["user-filter-1"] + +black-white-list: # compatible with deprecated config + instance: + do-dbs: ["all_mode"] + ignore-dbs: ["ignore_db"] + +filters: # filter rules, mysql instance can ref rules in it + user-filter-1: + schema-pattern: "*" # pattern of the upstream schema name, wildcard characters (*?) are supported + table-pattern: "*" # pattern of the upstream table name, wildcard characters (*?) are supported + events: ["truncate table", "drop table"] # ignore truncate/drop table ddl + action: Ignore + +mydumpers: + global: + threads: 4 + chunk-filesize: 64 + skip-tz-utc: true + extra-args: "" + +loaders: + global: + pool-size: 16 + dir: "./dumped_data" + +syncers: + global: + worker-count: 16 + batch: 100 + enable-ansi-quotes: false # compatible with deprecated config diff --git a/tests/lightning_mode/conf/dm-worker1.toml b/tests/lightning_mode/conf/dm-worker1.toml new file mode 100644 index 0000000000..7a72ea72bf --- /dev/null +++ b/tests/lightning_mode/conf/dm-worker1.toml @@ -0,0 +1,2 @@ +name = "worker1" +join = "127.0.0.1:8261" diff --git a/tests/lightning_mode/conf/dm-worker2.toml b/tests/lightning_mode/conf/dm-worker2.toml new file mode 100644 index 0000000000..010e21c73e --- /dev/null +++ b/tests/lightning_mode/conf/dm-worker2.toml @@ -0,0 +1,2 @@ +name = "worker2" +join = "127.0.0.1:8261" diff --git a/tests/lightning_mode/conf/source1.yaml b/tests/lightning_mode/conf/source1.yaml new file mode 100644 index 0000000000..664e2509c5 --- /dev/null +++ b/tests/lightning_mode/conf/source1.yaml @@ -0,0 +1,18 @@ +# MySQL Configuration. + +source-id: mysql-replica-01 +flavor: '' +enable-gtid: true +enable-relay: true +relay-binlog-name: '' +relay-binlog-gtid: '' +from: + host: 127.0.0.1 + user: root + password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs= + port: 3306 +checker: + check-enable: true + backoff-rollback: 5m + backoff-max: 5m + diff --git a/tests/lightning_mode/conf/source2.yaml b/tests/lightning_mode/conf/source2.yaml new file mode 100644 index 0000000000..32f44eb948 --- /dev/null +++ b/tests/lightning_mode/conf/source2.yaml @@ -0,0 +1,21 @@ +# MySQL Configuration. + +source-id: mysql-replica-02 +flavor: '' +enable-gtid: true +enable-relay: false +relay-binlog-name: '' +relay-binlog-gtid: '' +from: + host: 127.0.0.1 + user: root + password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs= + port: 3307 + + +# let dm-worker2 use the default config for checker +#checker: +# check-enable: true +# backoff-rollback: 5m +# backoff-max: 5m + diff --git a/tests/lightning_mode/data/db1.increment.sql b/tests/lightning_mode/data/db1.increment.sql new file mode 100644 index 0000000000..4509ea3a3a --- /dev/null +++ b/tests/lightning_mode/data/db1.increment.sql @@ -0,0 +1,48 @@ +use all_mode; +insert into t1 (id, name) values (3, 'Eddard Stark'); +update t1 set name = 'Arya Stark' where id = 1; +update t1 set name = 'Catelyn Stark' where name = 'catelyn'; + +-- test multi column index with generated column +alter table t1 add column info json; +alter table t1 add column gen_id int as (info->'$.id'); +alter table t1 add index multi_col(`id`, `gen_id`, ts); +insert into t1 (id, name, info) values (4, 'gentest', '{"id": 123}'); +insert into t1 (id, name, info) values (5, 'gentest', '{"id": 124}'); +update t1 set info = '{"id": 120}', ts = '2021-05-11 12:02:05' where id = 1; +update t1 set info = '{"id": 121}' where id = 2; +update t1 set info = '{"id": 122}' where id = 3; + +-- test genColumnCache is reset after ddl +alter table t1 add column info2 varchar(40); +insert into t1 (id, name, info) values (6, 'gentest', '{"id": 125, "test cache": false}'); +alter table t1 add unique key gen_idx(`gen_id`); +update t1 set name = 'gentestxx' where gen_id = 123; + +insert into t1 (id, name, info) values (7, 'gentest', '{"id": 126}'); +update t1 set name = 'gentestxxxxxx', dt = '2021-05-11 12:03:05', ts = '2021-05-11 12:03:05' where gen_id = 124; +-- delete with unique key +delete from t1 where gen_id > 124; + +-- test alter database +-- tidb doesn't support alter character set from latin1 to utf8m64 so we comment this now +-- alter database all_mode CHARACTER SET = utf8mb4; + +-- test decimal type +alter table t1 add column lat decimal(9,6) default '0.000000'; +insert into t1 (id, name, info, lat) values (8, 'gentest', '{"id":127}', '123.123'); + +-- test bit type +alter table t1 add column bin bit(1) default NULL; +insert into t1 (id, name, info, lat, bin) values (9, 'gentest', '{"id":128}', '123.123', b'0'); +insert into t1 (id, name, info, lat, bin) values (10, 'gentest', '{"id":129}', '123.123', b'1'); + +-- test bigint, min and max value for bigint/bigint unsigned +alter table t1 add column big1 bigint; +alter table t1 add column big2 bigint unsigned; +insert into t1 (id, name, info, lat, big1, big2) values (11, 'gentest', '{"id":130}', '123.123', -9223372036854775808, 0); +insert into t1 (id, name, info, lat, big1, big2) values (12, 'gentest', '{"id":131}', '123.123', 9223372036854775807, 18446744073709551615); + +-- test with different session time_zone +SET @@session.time_zone = '+07:00'; +insert into t1 (id, name, info) values (13, 'tztest', '{"id": 132}'); diff --git a/tests/lightning_mode/data/db1.increment0.sql b/tests/lightning_mode/data/db1.increment0.sql new file mode 100644 index 0000000000..fa406496ae --- /dev/null +++ b/tests/lightning_mode/data/db1.increment0.sql @@ -0,0 +1,3 @@ +SET @@SESSION.TIME_ZONE = "+03:00"; +use all_mode; +insert into t1 (id, name, ts) values (100, 'Eddard Stark', '2021-05-11 12:01:05'); diff --git a/tests/lightning_mode/data/db1.prepare.sql b/tests/lightning_mode/data/db1.prepare.sql new file mode 100644 index 0000000000..eef8893e84 --- /dev/null +++ b/tests/lightning_mode/data/db1.prepare.sql @@ -0,0 +1,20 @@ +drop database if exists `all_mode`; +create database `all_mode`; +use `all_mode`; +create table t1 ( + id int NOT NULL AUTO_INCREMENT, + name varchar(20), + dt datetime, + ts timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id)); +-- test ANSI_QUOTES works with quote in string +insert into t1 (id, name, dt, ts) values (1, 'ar"ya', now(), now()), (2, 'catelyn', '2021-05-11 10:01:05', '2021-05-11 10:01:05'); + +-- test sql_mode=NO_AUTO_VALUE_ON_ZERO +insert into t1 (id, name) values (0, 'lalala'); + +-- test block-allow-list +drop database if exists `ignore_db`; +create database `ignore_db`; +use `ignore_db`; +create table `ignore_table`(id int); \ No newline at end of file diff --git a/tests/lightning_mode/data/db2.increment.sql b/tests/lightning_mode/data/db2.increment.sql new file mode 100644 index 0000000000..bce1755e76 --- /dev/null +++ b/tests/lightning_mode/data/db2.increment.sql @@ -0,0 +1,5 @@ +use all_mode; +delete from t2 where name = 'Sansa'; + +-- test sql_mode=NO_AUTO_VALUE_ON_ZERO +insert into t2 (id, name) values (0,'haha') \ No newline at end of file diff --git a/tests/lightning_mode/data/db2.increment0.sql b/tests/lightning_mode/data/db2.increment0.sql new file mode 100644 index 0000000000..8ac48bc7a9 --- /dev/null +++ b/tests/lightning_mode/data/db2.increment0.sql @@ -0,0 +1 @@ +delete from all_mode.t2 where id = 1; diff --git a/tests/lightning_mode/data/db2.prepare.sql b/tests/lightning_mode/data/db2.prepare.sql new file mode 100644 index 0000000000..fc8900ae73 --- /dev/null +++ b/tests/lightning_mode/data/db2.prepare.sql @@ -0,0 +1,15 @@ +drop database if exists `all_mode`; +create database `all_mode`; +use `all_mode`; +create table t2 ( + id int NOT NULL AUTO_INCREMENT, + name varchar(20), + ts timestamp, + PRIMARY KEY (id));; +insert into t2 (name, ts) values ('Arya', now()), ('Bran', '2021-05-11 10:01:05'), ('Sansa', NULL); + +-- test block-allow-list +drop database if exists `ignore_db`; +create database `ignore_db`; +use `ignore_db`; +create table `ignore_table`(id int); \ No newline at end of file diff --git a/tests/lightning_mode/run.sh b/tests/lightning_mode/run.sh new file mode 100755 index 0000000000..bbdcb626f6 --- /dev/null +++ b/tests/lightning_mode/run.sh @@ -0,0 +1,259 @@ +#!/bin/bash + +set -eu + +cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source $cur/../_utils/test_prepare +WORK_DIR=$TEST_DIR/$TEST_NAME +API_VERSION="v1alpha1" +ILLEGAL_CHAR_NAME='t-Ë!s`t' + +function test_stop_task_before_checkpoint() { + run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 + check_contains 'Query OK, 2 rows affected' + run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 + check_contains 'Query OK, 3 rows affected' + + # start DM worker and master + run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml + check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT + check_metric $MASTER_PORT 'start_leader_counter' 3 0 2 + + export GO_FAILPOINTS='github.com/pingcap/dm/loader/WaitLoaderStopAfterInitCheckpoint=return(5)' + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + + # operate mysql config to worker + cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml + cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml + sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml + sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml + dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 + dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 + + # generate uncomplete checkpoint + dmctl_start_task "$cur/conf/dm-task.yaml" "--remove-meta" + check_log_contain_with_retry 'wait loader stop after init checkpoint' $WORK_DIR/worker1/log/dm-worker.log + check_log_contain_with_retry 'wait loader stop after init checkpoint' $WORK_DIR/worker2/log/dm-worker.log + run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "stop-task test" \ + "\"result\": true" 3 + + # restart dm-worker + pkill -9 dm-worker.test 2>/dev/null || true + check_port_offline $WORKER1_PORT 20 + check_port_offline $WORKER2_PORT 20 + + export GO_FAILPOINTS='github.com/pingcap/dm/loader/WaitLoaderStopBeforeLoadCheckpoint=return(5)' + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + + # stop-task before load checkpoint + dmctl_start_task "$cur/conf/dm-task.yaml" + check_log_contain_with_retry 'wait loader stop before load checkpoint' $WORK_DIR/worker1/log/dm-worker.log + check_log_contain_with_retry 'wait loader stop before load checkpoint' $WORK_DIR/worker2/log/dm-worker.log + run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "stop-task test" \ + "\"result\": true" 3 + + dmctl_start_task "$cur/conf/dm-task.yaml" + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml + run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "stop-task test" \ + "\"result\": true" 3 + + cleanup_data all_mode + cleanup_process $* + + export GO_FAILPOINTS='' +} + +function run() { + run_sql_both_source "SET @@GLOBAL.SQL_MODE='ANSI_QUOTES,NO_AUTO_VALUE_ON_ZERO'" + run_sql_source1 "SET @@global.time_zone = '+01:00';" + run_sql_source2 "SET @@global.time_zone = '+02:00';" + + inject_points=( + "github.com/pingcap/dm/dm/worker/TaskCheckInterval=return(\"500ms\")" + "github.com/pingcap/dm/relay/NewUpstreamServer=return(true)" + ) + export GO_FAILPOINTS="$(join_string \; ${inject_points[@]})" + + run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 + check_contains 'Query OK, 2 rows affected' + run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 + check_contains 'Query OK, 3 rows affected' + + # start DM worker and master + run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml + check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT + check_metric $MASTER_PORT 'start_leader_counter' 3 0 2 + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + + # operate mysql config to worker + cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml + cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml + sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml + sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml + # make sure source1 is bound to worker1 + dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 + + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "start-relay -s $SOURCE_ID1 worker1" \ + "\"result\": true" 1 + + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 + + # start DM task only + cp $cur/conf/dm-task.yaml $WORK_DIR/dm-task.yaml + sed -i "s/name: test/name: $ILLEGAL_CHAR_NAME/g" $WORK_DIR/dm-task.yaml + dmctl_start_task "$WORK_DIR/dm-task.yaml" "--remove-meta" + # check task has started + check_metric $WORKER1_PORT "dm_worker_task_state{source_id=\"mysql-replica-01\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker1\"}" 10 1 3 + check_metric $WORKER2_PORT "dm_worker_task_state{source_id=\"mysql-replica-02\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker2\"}" 10 1 3 + + # use sync_diff_inspector to check full dump loader + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml + + # check default session config + check_log_contain_with_retry '\\"tidb_txn_mode\\":\\"optimistic\\"' $WORK_DIR/worker1/log/dm-worker.log + check_log_contain_with_retry '\\"tidb_txn_mode\\":\\"optimistic\\"' $WORK_DIR/worker2/log/dm-worker.log + + # restart dm-worker1 + pkill -hup -f dm-worker1.toml 2>/dev/null || true + wait_pattern_exit dm-worker1.toml + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + # make sure worker1 have bound a source, and the source should same with bound before + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $ILLEGAL_CHAR_NAME" \ + "worker1" 1 + + # restart dm-worker2 + pkill -hup -f dm-worker2.toml 2>/dev/null || true + wait_pattern_exit dm-worker2.toml + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + check_metric $WORKER1_PORT "dm_worker_task_state{source_id=\"mysql-replica-01\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker1\"}" 10 1 3 + check_metric $WORKER2_PORT "dm_worker_task_state{source_id=\"mysql-replica-02\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker2\"}" 10 1 3 + + sleep 10 + echo "after restart dm-worker, task should resume automatically" + run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "start-task $WORK_DIR/dm-task.yaml" \ + "\"result\": false" 1 \ + "subtasks with name $ILLEGAL_CHAR_NAME for sources \[mysql-replica-01 mysql-replica-02\] already exist" 1 + sleep 2 + + # wait for task running + check_http_alive 127.0.0.1:$MASTER_PORT/apis/${API_VERSION}/status/$ILLEGAL_CHAR_NAME '"stage": "Running"' 10 + sleep 2 # still wait for subtask running on other dm-workers + + # kill tidb + pkill -hup tidb-server 2>/dev/null || true + wait_process_exit tidb-server + + # dm-worker execute sql failed, and will try auto resume task + run_sql_file $cur/data/db2.increment0.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 + sleep 2 + check_log_contains $WORK_DIR/worker2/log/dm-worker.log "dispatch auto resume task" + check_log_contains $WORK_DIR/worker2/log/dm-worker.log "lightning" + + # restart tidb, and task will recover success + run_tidb_server 4000 $TIDB_PASSWORD + sleep 2 + + # test after pause and resume relay, relay could continue from syncer's checkpoint + run_sql_source1 "flush logs" + run_sql_file $cur/data/db1.increment0.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 + + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml + + run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "pause-relay -s mysql-replica-01" \ + "\"result\": true" 2 + # we used failpoint to imitate an upstream switching, which purged whole relay dir + run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 + run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 + run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "resume-relay -s mysql-replica-01" \ + "\"result\": true" 2 + + sleep 2 + # relay should continue pulling from syncer's checkpoint, so only pull the latest binlog + server_uuid=$(tail -n 1 $WORK_DIR/worker1/relay_log/server-uuid.index) + echo "relay logs $(ls $WORK_DIR/worker1/relay_log/$server_uuid)" + relay_log_num=$(ls $WORK_DIR/worker1/relay_log/$server_uuid | grep -v 'relay.meta' | wc -l) + [ $relay_log_num -eq 1 ] + + # use sync_diff_inspector to check data now! + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml + + # test block-allow-list by the way + run_sql "show databases;" $TIDB_PORT $TIDB_PASSWORD + check_not_contains "ignore_db" + check_contains "all_mode" + + echo "check dump files have been cleaned" + ls $WORK_DIR/worker1/dumped_data.$ILLEGAL_CHAR_NAME && exit 1 || echo "worker1 auto removed dump files" + ls $WORK_DIR/worker2/dumped_data.$ILLEGAL_CHAR_NAME && exit 1 || echo "worker2 auto removed dump files" + + echo "check no password in log" + check_log_not_contains $WORK_DIR/master/log/dm-master.log "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=" + check_log_not_contains $WORK_DIR/worker1/log/dm-worker.log "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=" + check_log_not_contains $WORK_DIR/worker2/log/dm-worker.log "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=" + check_log_not_contains $WORK_DIR/master/log/dm-master.log "123456" + check_log_not_contains $WORK_DIR/worker1/log/dm-worker.log "123456" + check_log_not_contains $WORK_DIR/worker2/log/dm-worker.log "123456" + + # test drop table if exists + run_sql_source1 "drop table if exists \`all_mode\`.\`tb1\`;" + run_sql_source1 "drop table if exists \`all_mode\`.\`tb1\`;" + run_sql_source2 "drop table if exists \`all_mode\`.\`tb2\`;" + run_sql_source2 "drop table if exists \`all_mode\`.\`tb2\`;" + check_log_not_contains $WORK_DIR/worker1/log/dm-worker.log "Error .* Table .* doesn't exist" + check_log_not_contains $WORK_DIR/worker2/log/dm-worker.log "Error .* Table .* doesn't exist" + + # test Db not exists should be reported + + run_sql_tidb "drop database all_mode" + run_sql_source1 "create table all_mode.db_error (c int primary key);" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $ILLEGAL_CHAR_NAME" \ + "Error 1049: Unknown database" 1 + + # stop task, task state should be cleaned + run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "stop-task $ILLEGAL_CHAR_NAME" \ + "\"result\": true" 3 + check_metric_not_contains $WORKER1_PORT "dm_worker_task_state{source_id=\"mysql-replica-01\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker1\"}" 3 + check_metric_not_contains $WORKER2_PORT "dm_worker_task_state{source_id=\"mysql-replica-02\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker2\"}" 3 + + # all unit without error. + check_metric_not_contains $WORKER1_PORT "dm_mydumper_exit_with_error_count" 3 + check_metric_not_contains $WORKER1_PORT "dm_loader_exit_with_error_count" 3 + check_metric_not_contains $WORKER1_PORT "dm_syncer_exit_with_error_count" 3 + + # check syncer metrics + check_two_metric_equal $WORKER1_PORT 'dm_syncer_binlog_file{node="master"' 'dm_syncer_binlog_file{node="syncer"' 3 + check_two_metric_equal $WORKER2_PORT 'dm_syncer_binlog_file{node="master"' 'dm_syncer_binlog_file{node="syncer"' 3 + export GO_FAILPOINTS='' + + run_sql_both_source "SET @@GLOBAL.SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'" + run_sql_both_source "SET @@global.time_zone = 'SYSTEM';" +} + +cleanup_data all_mode +# also cleanup dm processes in case of last run failed +cleanup_process $* +run $* +cleanup_process $* + +echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>" From 8026c1e7214715b809c50fa9531cba2c0e7736b3 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Tue, 28 Sep 2021 01:18:49 +0800 Subject: [PATCH 11/23] fix comment Signed-off-by: Little-Wallace --- tests/dmctl_command/conf/dm-task.yaml | 1 + tests/import_v10x/conf/task.yaml | 1 + tests/lightning_mode/conf/dm-task.yaml | 3 +++ tests/lightning_mode/run.sh | 18 +++++++++--------- tests/new_relay/configs/tasks/test.yaml | 1 + 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/dmctl_command/conf/dm-task.yaml b/tests/dmctl_command/conf/dm-task.yaml index 21b6540b32..702e88919e 100644 --- a/tests/dmctl_command/conf/dm-task.yaml +++ b/tests/dmctl_command/conf/dm-task.yaml @@ -7,6 +7,7 @@ meta-schema: "dm_meta" heartbeat-update-interval: 1 heartbeat-report-interval: 1 remove-meta: true +tidb: null target-database: host: "127.0.0.1" diff --git a/tests/import_v10x/conf/task.yaml b/tests/import_v10x/conf/task.yaml index a71d12b9d1..0676560aed 100644 --- a/tests/import_v10x/conf/task.yaml +++ b/tests/import_v10x/conf/task.yaml @@ -114,3 +114,4 @@ syncers: clean-dump-file: false ansi-quotes: false remove-meta: false +tidb: null diff --git a/tests/lightning_mode/conf/dm-task.yaml b/tests/lightning_mode/conf/dm-task.yaml index ddc192017f..5dd7a7cbbd 100644 --- a/tests/lightning_mode/conf/dm-task.yaml +++ b/tests/lightning_mode/conf/dm-task.yaml @@ -37,6 +37,9 @@ black-white-list: # compatible with deprecated config do-dbs: ["all_mode"] ignore-dbs: ["ignore_db"] +tidb: + backend: "tidb" + filters: # filter rules, mysql instance can ref rules in it user-filter-1: schema-pattern: "*" # pattern of the upstream schema name, wildcard characters (*?) are supported diff --git a/tests/lightning_mode/run.sh b/tests/lightning_mode/run.sh index bbdcb626f6..6a6fe9be80 100755 --- a/tests/lightning_mode/run.sh +++ b/tests/lightning_mode/run.sh @@ -66,7 +66,7 @@ function test_stop_task_before_checkpoint() { "stop-task test" \ "\"result\": true" 3 - cleanup_data all_mode + cleanup_data lightning_mode cleanup_process $* export GO_FAILPOINTS='' @@ -199,7 +199,7 @@ function run() { # test block-allow-list by the way run_sql "show databases;" $TIDB_PORT $TIDB_PASSWORD check_not_contains "ignore_db" - check_contains "all_mode" + check_contains "lightning_mode" echo "check dump files have been cleaned" ls $WORK_DIR/worker1/dumped_data.$ILLEGAL_CHAR_NAME && exit 1 || echo "worker1 auto removed dump files" @@ -214,17 +214,17 @@ function run() { check_log_not_contains $WORK_DIR/worker2/log/dm-worker.log "123456" # test drop table if exists - run_sql_source1 "drop table if exists \`all_mode\`.\`tb1\`;" - run_sql_source1 "drop table if exists \`all_mode\`.\`tb1\`;" - run_sql_source2 "drop table if exists \`all_mode\`.\`tb2\`;" - run_sql_source2 "drop table if exists \`all_mode\`.\`tb2\`;" + run_sql_source1 "drop table if exists \`lightning_mode\`.\`tb1\`;" + run_sql_source1 "drop table if exists \`lightning_mode\`.\`tb1\`;" + run_sql_source2 "drop table if exists \`lightning_mode\`.\`tb2\`;" + run_sql_source2 "drop table if exists \`lightning_mode\`.\`tb2\`;" check_log_not_contains $WORK_DIR/worker1/log/dm-worker.log "Error .* Table .* doesn't exist" check_log_not_contains $WORK_DIR/worker2/log/dm-worker.log "Error .* Table .* doesn't exist" # test Db not exists should be reported - run_sql_tidb "drop database all_mode" - run_sql_source1 "create table all_mode.db_error (c int primary key);" + run_sql_tidb "drop database lightning_mode" + run_sql_source1 "create table lightning_mode.db_error (c int primary key);" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status $ILLEGAL_CHAR_NAME" \ "Error 1049: Unknown database" 1 @@ -250,7 +250,7 @@ function run() { run_sql_both_source "SET @@global.time_zone = 'SYSTEM';" } -cleanup_data all_mode +cleanup_data lightning_mode # also cleanup dm processes in case of last run failed cleanup_process $* run $* diff --git a/tests/new_relay/configs/tasks/test.yaml b/tests/new_relay/configs/tasks/test.yaml index 17b0698cfc..79c130b9ef 100644 --- a/tests/new_relay/configs/tasks/test.yaml +++ b/tests/new_relay/configs/tasks/test.yaml @@ -59,6 +59,7 @@ syncers: clean-dump-file: false ansi-quotes: false remove-meta: false +tidb: null mysql-instances: - source-id: mysql-replica-01 meta: null From 9dd6693c355cbba55ae67a4adc4baf75d27ab073 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Tue, 28 Sep 2021 10:55:44 +0800 Subject: [PATCH 12/23] fix test Signed-off-by: Little-Wallace --- loader/lightning.go | 13 +++++++------ tests/new_relay/configs/tasks/test.yaml | 1 - 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/loader/lightning.go b/loader/lightning.go index 51dfa22ba8..e76dede5e9 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -1,4 +1,4 @@ -// Copyright 2019 PingCAP, Inc. +// Copyright 2021 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,10 +18,6 @@ import ( "path/filepath" "sync" - "github.com/docker/go-units" - "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb/br/pkg/lightning/common" - "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/dm/pb" "github.com/pingcap/dm/dm/unit" @@ -31,8 +27,11 @@ import ( "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/utils" + "github.com/docker/go-units" "github.com/pingcap/failpoint" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/br/pkg/lightning" + "github.com/pingcap/tidb/br/pkg/lightning/common" lcfg "github.com/pingcap/tidb/br/pkg/lightning/config" "go.etcd.io/etcd/clientv3" "go.uber.org/atomic" @@ -179,7 +178,9 @@ func (l *LightningLoader) restore(ctx context.Context) error { l.finish.Store(true) } if l.cfg.Mode == config.ModeFull { - _ = delLoadTask(l.cli, l.cfg, l.workerName) + if err := delLoadTask(l.cli, l.cfg, l.workerName); err != nil { + return err + } } if l.finish.Load() { if l.cfg.CleanDumpFile { diff --git a/tests/new_relay/configs/tasks/test.yaml b/tests/new_relay/configs/tasks/test.yaml index 79c130b9ef..17b0698cfc 100644 --- a/tests/new_relay/configs/tasks/test.yaml +++ b/tests/new_relay/configs/tasks/test.yaml @@ -59,7 +59,6 @@ syncers: clean-dump-file: false ansi-quotes: false remove-meta: false -tidb: null mysql-instances: - source-id: mysql-replica-01 meta: null From 0bbaeb49e782c6cc44b5b0526afce7ec7a3ea564 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Tue, 28 Sep 2021 12:25:44 +0800 Subject: [PATCH 13/23] fix config test Signed-off-by: Little-Wallace --- dm/config/task_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/config/task_test.go b/dm/config/task_test.go index bf25b73efd..725075e2be 100644 --- a/dm/config/task_test.go +++ b/dm/config/task_test.go @@ -1026,7 +1026,7 @@ func (t *testConfig) TestTaskConfigForDowngrade(c *C) { // make sure all new field were added cfgReflect := reflect.Indirect(reflect.ValueOf(cfg)) cfgForDowngradeReflect := reflect.Indirect(reflect.ValueOf(cfgForDowngrade)) - c.Assert(cfgReflect.NumField(), Equals, cfgForDowngradeReflect.NumField()+1) // without flag + c.Assert(cfgReflect.NumField(), Equals, cfgForDowngradeReflect.NumField()+2) // without flag and tidb // make sure all field were copied cfgForClone := &TaskConfigForDowngrade{} From 6b0dc8cc8a7cdc7fdb0803ca3be328fdbcba9870 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Tue, 28 Sep 2021 12:45:05 +0800 Subject: [PATCH 14/23] fix fmt Signed-off-by: Little-Wallace --- dm/config/subtask.go | 3 +-- loader/lightning.go | 2 +- tests/others_integration_1.txt | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dm/config/subtask.go b/dm/config/subtask.go index 37ee98c288..30af32f836 100644 --- a/dm/config/subtask.go +++ b/dm/config/subtask.go @@ -458,10 +458,9 @@ func (c *SubTaskConfig) Adjust(verifyDecryptPassword bool) error { return terror.ErrConfigInvalidChunkFileSize.Generate(c.MydumperConfig.ChunkFilesize) } - if c.TiDB.Backend != "" && c.TiDB.Backend != lcfg.BackendLocal && c.TiDB.Backend != lcfg.BackendTiDB { return terror.ErrLoadBackendNotSupport.Generate(c.TiDB.Backend) - } + } if _, err := bf.NewBinlogEvent(c.CaseSensitive, c.FilterRules); err != nil { return terror.ErrConfigBinlogEventFilter.Delegate(err) } diff --git a/loader/lightning.go b/loader/lightning.go index e76dede5e9..938ec61656 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -178,7 +178,7 @@ func (l *LightningLoader) restore(ctx context.Context) error { l.finish.Store(true) } if l.cfg.Mode == config.ModeFull { - if err := delLoadTask(l.cli, l.cfg, l.workerName); err != nil { + if err = delLoadTask(l.cli, l.cfg, l.workerName); err != nil { return err } } diff --git a/tests/others_integration_1.txt b/tests/others_integration_1.txt index f5e9656415..d5361c58d4 100644 --- a/tests/others_integration_1.txt +++ b/tests/others_integration_1.txt @@ -7,3 +7,4 @@ gtid only_dml adjust_gtid checkpoint_transaction +lightning_mode From ac64b6fd3a8ff5c58a4ca880c5974531a4d19b18 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Tue, 28 Sep 2021 14:31:30 +0800 Subject: [PATCH 15/23] fix db name Signed-off-by: Little-Wallace --- tests/lightning_mode/conf/diff_config.toml | 10 +++++----- tests/lightning_mode/conf/dm-task.yaml | 2 +- tests/lightning_mode/data/db1.increment.sql | 4 ++-- tests/lightning_mode/data/db1.increment0.sql | 2 +- tests/lightning_mode/data/db1.prepare.sql | 6 +++--- tests/lightning_mode/data/db2.increment.sql | 2 +- tests/lightning_mode/data/db2.increment0.sql | 2 +- tests/lightning_mode/data/db2.prepare.sql | 6 +++--- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/lightning_mode/conf/diff_config.toml b/tests/lightning_mode/conf/diff_config.toml index 6dfc688aaf..76407db174 100644 --- a/tests/lightning_mode/conf/diff_config.toml +++ b/tests/lightning_mode/conf/diff_config.toml @@ -14,25 +14,25 @@ fix-sql-file = "fix.sql" # tables need to check. [[check-tables]] -schema = "all_mode" +schema = "lightning_mode" tables = ["~t.*"] [[table-config]] -schema = "all_mode" +schema = "lightning_mode" table = "t1" [[table-config.source-tables]] instance-id = "source-1" -schema = "all_mode" +schema = "lightning_mode" table = "t1" [[table-config]] -schema = "all_mode" +schema = "lightning_mode" table = "t2" [[table-config.source-tables]] instance-id = "source-2" -schema = "all_mode" +schema = "lightning_mode" table = "t2" [[source-db]] diff --git a/tests/lightning_mode/conf/dm-task.yaml b/tests/lightning_mode/conf/dm-task.yaml index 5dd7a7cbbd..2b3d43261c 100644 --- a/tests/lightning_mode/conf/dm-task.yaml +++ b/tests/lightning_mode/conf/dm-task.yaml @@ -34,7 +34,7 @@ mysql-instances: black-white-list: # compatible with deprecated config instance: - do-dbs: ["all_mode"] + do-dbs: ["lightning_mode"] ignore-dbs: ["ignore_db"] tidb: diff --git a/tests/lightning_mode/data/db1.increment.sql b/tests/lightning_mode/data/db1.increment.sql index 4509ea3a3a..9d7ebc4879 100644 --- a/tests/lightning_mode/data/db1.increment.sql +++ b/tests/lightning_mode/data/db1.increment.sql @@ -1,4 +1,4 @@ -use all_mode; +use lightning_mode; insert into t1 (id, name) values (3, 'Eddard Stark'); update t1 set name = 'Arya Stark' where id = 1; update t1 set name = 'Catelyn Stark' where name = 'catelyn'; @@ -26,7 +26,7 @@ delete from t1 where gen_id > 124; -- test alter database -- tidb doesn't support alter character set from latin1 to utf8m64 so we comment this now --- alter database all_mode CHARACTER SET = utf8mb4; +-- alter database lightning_mode CHARACTER SET = utf8mb4; -- test decimal type alter table t1 add column lat decimal(9,6) default '0.000000'; diff --git a/tests/lightning_mode/data/db1.increment0.sql b/tests/lightning_mode/data/db1.increment0.sql index fa406496ae..b1f2fcd40f 100644 --- a/tests/lightning_mode/data/db1.increment0.sql +++ b/tests/lightning_mode/data/db1.increment0.sql @@ -1,3 +1,3 @@ SET @@SESSION.TIME_ZONE = "+03:00"; -use all_mode; +use lightning_mode; insert into t1 (id, name, ts) values (100, 'Eddard Stark', '2021-05-11 12:01:05'); diff --git a/tests/lightning_mode/data/db1.prepare.sql b/tests/lightning_mode/data/db1.prepare.sql index eef8893e84..384c9d414c 100644 --- a/tests/lightning_mode/data/db1.prepare.sql +++ b/tests/lightning_mode/data/db1.prepare.sql @@ -1,6 +1,6 @@ -drop database if exists `all_mode`; -create database `all_mode`; -use `all_mode`; +drop database if exists `lightning_mode`; +create database `lightning_mode`; +use `lightning_mode`; create table t1 ( id int NOT NULL AUTO_INCREMENT, name varchar(20), diff --git a/tests/lightning_mode/data/db2.increment.sql b/tests/lightning_mode/data/db2.increment.sql index bce1755e76..c3df853e84 100644 --- a/tests/lightning_mode/data/db2.increment.sql +++ b/tests/lightning_mode/data/db2.increment.sql @@ -1,4 +1,4 @@ -use all_mode; +use lightning_mode; delete from t2 where name = 'Sansa'; -- test sql_mode=NO_AUTO_VALUE_ON_ZERO diff --git a/tests/lightning_mode/data/db2.increment0.sql b/tests/lightning_mode/data/db2.increment0.sql index 8ac48bc7a9..2369062763 100644 --- a/tests/lightning_mode/data/db2.increment0.sql +++ b/tests/lightning_mode/data/db2.increment0.sql @@ -1 +1 @@ -delete from all_mode.t2 where id = 1; +delete from lightning_mode.t2 where id = 1; diff --git a/tests/lightning_mode/data/db2.prepare.sql b/tests/lightning_mode/data/db2.prepare.sql index fc8900ae73..cf25bc399c 100644 --- a/tests/lightning_mode/data/db2.prepare.sql +++ b/tests/lightning_mode/data/db2.prepare.sql @@ -1,6 +1,6 @@ -drop database if exists `all_mode`; -create database `all_mode`; -use `all_mode`; +drop database if exists `lightning_mode`; +create database `lightning_mode`; +use `lightning_mode`; create table t2 ( id int NOT NULL AUTO_INCREMENT, name varchar(20), From 6fb6334343fe06e0e709fa94ed7ad1ca0cba3146 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Tue, 28 Sep 2021 18:23:40 +0800 Subject: [PATCH 16/23] add sql mode Signed-off-by: Little-Wallace --- dm/worker/subtask.go | 6 +++++- loader/lightning.go | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/dm/worker/subtask.go b/dm/worker/subtask.go index 863c42f1dc..a01954da8f 100644 --- a/dm/worker/subtask.go +++ b/dm/worker/subtask.go @@ -68,7 +68,11 @@ func createRealUnits(cfg *config.SubTaskConfig, etcdClient *clientv3.Client, wor case config.ModeFull: // NOTE: maybe need another checker in the future? us = append(us, dumpling.NewDumpling(cfg)) - us = append(us, loader.NewLoader(cfg, etcdClient, workerName)) + if cfg.TiDB.Backend == "" { + us = append(us, loader.NewLoader(cfg, etcdClient, workerName)) + } else { + us = append(us, loader.NewLightning(cfg, etcdClient, workerName)) + } case config.ModeIncrement: us = append(us, syncer.NewSyncer(cfg, etcdClient)) default: diff --git a/loader/lightning.go b/loader/lightning.go index 938ec61656..fcb2ae0acb 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -160,6 +160,12 @@ func (l *LightningLoader) restore(ctx context.Context) error { TLS: cfg.TiDB.TLS, } cfg.Checkpoint.DSN = param.ToDSN() + sqlMode, err2 := mysql.GetSQLMode(l.cfg.LoaderConfig.SQLMode) + if err2 != nil { + l.logger.Warn("cannot convert sql_mode compatible", log.ShortError(err2)) + } else { + cfg.TiDB.SQLMode = sqlMode + } if err = cfg.Adjust(ctx); err != nil { return err } From 9fed14c16cf7b4708b8450b5856777c91b1a95c6 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Tue, 28 Sep 2021 20:09:37 +0800 Subject: [PATCH 17/23] fix meta Signed-off-by: Little-Wallace --- loader/lightning.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/loader/lightning.go b/loader/lightning.go index fcb2ae0acb..12a290efc8 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -148,7 +148,6 @@ func (l *LightningLoader) restore(ctx context.Context) error { return err } cfg.Routes = l.cfg.RouteRules - cfg.Checkpoint.Schema = "tidb_lightning_checkpoint_" + l.cfg.Name cfg.Checkpoint.Driver = lcfg.CheckpointDriverMySQL param := common.MySQLConnectParam{ Host: cfg.TiDB.Host, @@ -160,12 +159,7 @@ func (l *LightningLoader) restore(ctx context.Context) error { TLS: cfg.TiDB.TLS, } cfg.Checkpoint.DSN = param.ToDSN() - sqlMode, err2 := mysql.GetSQLMode(l.cfg.LoaderConfig.SQLMode) - if err2 != nil { - l.logger.Warn("cannot convert sql_mode compatible", log.ShortError(err2)) - } else { - cfg.TiDB.SQLMode = sqlMode - } + cfg.TiDB.StrSQLMode = l.cfg.LoaderConfig.SQLMode if err = cfg.Adjust(ctx); err != nil { return err } From 4aed64326ed19b86360832c8e81879bc25c29ed2 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Wed, 29 Sep 2021 15:36:26 +0800 Subject: [PATCH 18/23] remove timestamp Signed-off-by: Little-Wallace --- tests/lightning_mode/data/db1.increment.sql | 6 +- tests/lightning_mode/data/db1.increment0.sql | 3 +- tests/lightning_mode/data/db1.prepare.sql | 4 +- tests/lightning_mode/data/db2.prepare.sql | 3 +- tests/lightning_mode/run.sh | 67 -------------------- 5 files changed, 6 insertions(+), 77 deletions(-) diff --git a/tests/lightning_mode/data/db1.increment.sql b/tests/lightning_mode/data/db1.increment.sql index 9d7ebc4879..9c2f9be4a9 100644 --- a/tests/lightning_mode/data/db1.increment.sql +++ b/tests/lightning_mode/data/db1.increment.sql @@ -6,10 +6,10 @@ update t1 set name = 'Catelyn Stark' where name = 'catelyn'; -- test multi column index with generated column alter table t1 add column info json; alter table t1 add column gen_id int as (info->'$.id'); -alter table t1 add index multi_col(`id`, `gen_id`, ts); +alter table t1 add index multi_col(`id`, `gen_id`); insert into t1 (id, name, info) values (4, 'gentest', '{"id": 123}'); insert into t1 (id, name, info) values (5, 'gentest', '{"id": 124}'); -update t1 set info = '{"id": 120}', ts = '2021-05-11 12:02:05' where id = 1; +update t1 set info = '{"id": 120}' where id = 1; update t1 set info = '{"id": 121}' where id = 2; update t1 set info = '{"id": 122}' where id = 3; @@ -20,7 +20,7 @@ alter table t1 add unique key gen_idx(`gen_id`); update t1 set name = 'gentestxx' where gen_id = 123; insert into t1 (id, name, info) values (7, 'gentest', '{"id": 126}'); -update t1 set name = 'gentestxxxxxx', dt = '2021-05-11 12:03:05', ts = '2021-05-11 12:03:05' where gen_id = 124; +update t1 set name = 'gentestxxxxxx' where gen_id = 124; -- delete with unique key delete from t1 where gen_id > 124; diff --git a/tests/lightning_mode/data/db1.increment0.sql b/tests/lightning_mode/data/db1.increment0.sql index b1f2fcd40f..74df7be843 100644 --- a/tests/lightning_mode/data/db1.increment0.sql +++ b/tests/lightning_mode/data/db1.increment0.sql @@ -1,3 +1,2 @@ -SET @@SESSION.TIME_ZONE = "+03:00"; use lightning_mode; -insert into t1 (id, name, ts) values (100, 'Eddard Stark', '2021-05-11 12:01:05'); +insert into t1 (id, name) values (100, 'Eddard Stark'); diff --git a/tests/lightning_mode/data/db1.prepare.sql b/tests/lightning_mode/data/db1.prepare.sql index 384c9d414c..bcf4214400 100644 --- a/tests/lightning_mode/data/db1.prepare.sql +++ b/tests/lightning_mode/data/db1.prepare.sql @@ -4,11 +4,9 @@ use `lightning_mode`; create table t1 ( id int NOT NULL AUTO_INCREMENT, name varchar(20), - dt datetime, - ts timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (id)); -- test ANSI_QUOTES works with quote in string -insert into t1 (id, name, dt, ts) values (1, 'ar"ya', now(), now()), (2, 'catelyn', '2021-05-11 10:01:05', '2021-05-11 10:01:05'); +insert into t1 (id, name) values (1, 'ar"ya'), (2, 'catelyn'); -- test sql_mode=NO_AUTO_VALUE_ON_ZERO insert into t1 (id, name) values (0, 'lalala'); diff --git a/tests/lightning_mode/data/db2.prepare.sql b/tests/lightning_mode/data/db2.prepare.sql index cf25bc399c..548dc92c0d 100644 --- a/tests/lightning_mode/data/db2.prepare.sql +++ b/tests/lightning_mode/data/db2.prepare.sql @@ -4,9 +4,8 @@ use `lightning_mode`; create table t2 ( id int NOT NULL AUTO_INCREMENT, name varchar(20), - ts timestamp, PRIMARY KEY (id));; -insert into t2 (name, ts) values ('Arya', now()), ('Bran', '2021-05-11 10:01:05'), ('Sansa', NULL); +insert into t2 (name) values ('Arya'), ('Bran'), ('Sansa'); -- test block-allow-list drop database if exists `ignore_db`; diff --git a/tests/lightning_mode/run.sh b/tests/lightning_mode/run.sh index 6a6fe9be80..fc42a79fe9 100755 --- a/tests/lightning_mode/run.sh +++ b/tests/lightning_mode/run.sh @@ -8,74 +8,8 @@ WORK_DIR=$TEST_DIR/$TEST_NAME API_VERSION="v1alpha1" ILLEGAL_CHAR_NAME='t-Ë!s`t' -function test_stop_task_before_checkpoint() { - run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 - check_contains 'Query OK, 2 rows affected' - run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 - check_contains 'Query OK, 3 rows affected' - - # start DM worker and master - run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml - check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT - check_metric $MASTER_PORT 'start_leader_counter' 3 0 2 - - export GO_FAILPOINTS='github.com/pingcap/dm/loader/WaitLoaderStopAfterInitCheckpoint=return(5)' - run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml - check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT - run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml - check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT - - # operate mysql config to worker - cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml - cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml - sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml - sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml - dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 - dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 - - # generate uncomplete checkpoint - dmctl_start_task "$cur/conf/dm-task.yaml" "--remove-meta" - check_log_contain_with_retry 'wait loader stop after init checkpoint' $WORK_DIR/worker1/log/dm-worker.log - check_log_contain_with_retry 'wait loader stop after init checkpoint' $WORK_DIR/worker2/log/dm-worker.log - run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ - "stop-task test" \ - "\"result\": true" 3 - - # restart dm-worker - pkill -9 dm-worker.test 2>/dev/null || true - check_port_offline $WORKER1_PORT 20 - check_port_offline $WORKER2_PORT 20 - - export GO_FAILPOINTS='github.com/pingcap/dm/loader/WaitLoaderStopBeforeLoadCheckpoint=return(5)' - run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml - run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml - check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT - check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT - - # stop-task before load checkpoint - dmctl_start_task "$cur/conf/dm-task.yaml" - check_log_contain_with_retry 'wait loader stop before load checkpoint' $WORK_DIR/worker1/log/dm-worker.log - check_log_contain_with_retry 'wait loader stop before load checkpoint' $WORK_DIR/worker2/log/dm-worker.log - run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ - "stop-task test" \ - "\"result\": true" 3 - - dmctl_start_task "$cur/conf/dm-task.yaml" - check_sync_diff $WORK_DIR $cur/conf/diff_config.toml - run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ - "stop-task test" \ - "\"result\": true" 3 - - cleanup_data lightning_mode - cleanup_process $* - - export GO_FAILPOINTS='' -} - function run() { run_sql_both_source "SET @@GLOBAL.SQL_MODE='ANSI_QUOTES,NO_AUTO_VALUE_ON_ZERO'" - run_sql_source1 "SET @@global.time_zone = '+01:00';" - run_sql_source2 "SET @@global.time_zone = '+02:00';" inject_points=( "github.com/pingcap/dm/dm/worker/TaskCheckInterval=return(\"500ms\")" @@ -247,7 +181,6 @@ function run() { export GO_FAILPOINTS='' run_sql_both_source "SET @@GLOBAL.SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'" - run_sql_both_source "SET @@global.time_zone = 'SYSTEM';" } cleanup_data lightning_mode From df256a3c344a884ef3d0e4bf3f7e751397783494 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Mon, 11 Oct 2021 11:49:58 +0800 Subject: [PATCH 19/23] fix tablename quotes Signed-off-by: Little-Wallace --- dm/config/subtask.go | 1 + go.mod | 8 ++++---- go.sum | 29 +++++++++++++++++------------ loader/lightning.go | 2 ++ loader/util.go | 3 +-- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/dm/config/subtask.go b/dm/config/subtask.go index a13ba64cb2..927db70188 100644 --- a/dm/config/subtask.go +++ b/dm/config/subtask.go @@ -46,6 +46,7 @@ const ( ShadowTableRules = "shadow-table-rules" TrashTableRules = "trash-table-rules" + TiDBLightningCheckpointPrefix = "tidb_lightning_checkpoint_" ) var defaultMaxIdleConns = 2 diff --git a/go.mod b/go.mod index 8f4b604694..e091057367 100644 --- a/go.mod +++ b/go.mod @@ -25,8 +25,8 @@ require ( github.com/pingcap/errors v0.11.5-0.20210513014640-40f9a1999b3b github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 - github.com/pingcap/parser v0.0.0-20210915160042-658edc1dba6a - github.com/pingcap/tidb v1.1.0-beta.0.20210923162246-d14566c7ed3e + github.com/pingcap/parser v0.0.0-20211004012448-687005894c4e + github.com/pingcap/tidb v1.1.0-beta.0.20211011021924-2f9d591ef722 github.com/pingcap/tidb-tools v5.2.0-alpha.0.20210727084616-915b22e4d42c+incompatible github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.5.1 @@ -42,9 +42,9 @@ require ( github.com/tidwall/sjson v1.1.2 github.com/tikv/pd v1.1.0-beta.0.20210818082359-acba1da0018d github.com/unrolled/render v1.0.1 - go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b + go.etcd.io/etcd v0.5.0-alpha.5.0.20210512015243-d19fbe541bf9 go.uber.org/atomic v1.9.0 - go.uber.org/zap v1.19.0 + go.uber.org/zap v1.19.1 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 diff --git a/go.sum b/go.sum index dd62aebe37..3742438284 100644 --- a/go.sum +++ b/go.sum @@ -165,6 +165,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cznic/golex v0.0.0-20181122101858-9c343928389c/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= @@ -398,8 +399,9 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= @@ -642,16 +644,16 @@ github.com/pingcap/log v0.0.0-20210906054005-afc726e70354/go.mod h1:DWQW5jICDR7U github.com/pingcap/parser v0.0.0-20210415081931-48e7f467fd74/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= github.com/pingcap/parser v0.0.0-20210907051057-948434fa20e4/go.mod h1:Ek0mLKEqUGnQqBw1JnYrJQxsguU433DU68yUbsoeJ7s= -github.com/pingcap/parser v0.0.0-20210915160042-658edc1dba6a h1:dApxKqk49ZaE/oErh4h5W27SJyyy5/oywIzpStANgLI= -github.com/pingcap/parser v0.0.0-20210915160042-658edc1dba6a/go.mod h1:+xcMiiZzdIktT/Nqdfm81dkECJ2EPuoAYywd57py4Pk= +github.com/pingcap/parser v0.0.0-20211004012448-687005894c4e h1:dPMDpj+7ng9qEWoT3n6qjpB1ohz79uTLVM6ILW+ZMT0= +github.com/pingcap/parser v0.0.0-20211004012448-687005894c4e/go.mod h1:+xcMiiZzdIktT/Nqdfm81dkECJ2EPuoAYywd57py4Pk= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3/go.mod h1:tckvA041UWP+NqYzrJ3fMgC/Hw9wnmQ/tUkp/JaHly8= github.com/pingcap/sysutil v0.0.0-20210730114356-fcd8a63f68c5 h1:7rvAtZe/ZUzOKzgriNPQoBNvleJXBk4z7L3Z47+tS98= github.com/pingcap/sysutil v0.0.0-20210730114356-fcd8a63f68c5/go.mod h1:XsOaV712rUk63aOEKYP9PhXTIE3FMNHmC2r1wX5wElY= github.com/pingcap/tidb v1.1.0-beta.0.20210914080841-89617e20b2c0/go.mod h1:MSg0d2+iJI/d3mUOcLV82GLpw5+jNGq21Z8qcHqBI/g= github.com/pingcap/tidb v1.1.0-beta.0.20210914112841-6ebfe8aa4257/go.mod h1:MSg0d2+iJI/d3mUOcLV82GLpw5+jNGq21Z8qcHqBI/g= -github.com/pingcap/tidb v1.1.0-beta.0.20210923162246-d14566c7ed3e h1:2MFgYfLNHoEMhUw0Nx6gRhh+qK7rADQl5V2aTTDc088= -github.com/pingcap/tidb v1.1.0-beta.0.20210923162246-d14566c7ed3e/go.mod h1:EWxcIyryh3CTwyuzlqllMZhrgxFbjJkUmx9/UXN0GVE= +github.com/pingcap/tidb v1.1.0-beta.0.20211011021924-2f9d591ef722 h1:Cb5Tk1tadtPpMu+za8L3JY/tDBe2zoY+EObwvElYVnU= +github.com/pingcap/tidb v1.1.0-beta.0.20211011021924-2f9d591ef722/go.mod h1:WQHMalPfxjTrWMSSsKsPaJsM4ErDr2/ZOF4Gpd70CGQ= github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo= github.com/pingcap/tidb-dashboard v0.0.0-20210716172320-2226872e3296/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ= github.com/pingcap/tidb-tools v5.0.3+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= @@ -798,8 +800,8 @@ github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ= github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210913094925-a8fa8acd44e7/go.mod h1:KwtZXt0JD+bP9bWW2ka0ir3Wp3oTEfZUTh22bs2sI4o= -github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210918071319-6ca8098ae0fe h1:nOeupIzlqbMP/UswMxfqvz2YftFa2x+N13QN0QpyiZs= -github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210918071319-6ca8098ae0fe/go.mod h1:KwtZXt0JD+bP9bWW2ka0ir3Wp3oTEfZUTh22bs2sI4o= +github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210926100628-3cc2459779ca h1:aE/y7lkygl716M6eIos95BKjM1mGew+EoLWRQAXDoAE= +github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210926100628-3cc2459779ca/go.mod h1:KwtZXt0JD+bP9bWW2ka0ir3Wp3oTEfZUTh22bs2sI4o= github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d/go.mod h1:Jw9KG11C/23Rr7DW4XWQ7H5xOgGZo6DFL1OKAF4+Igw= github.com/tikv/pd v1.1.0-beta.0.20210818082359-acba1da0018d h1:AFm1Dzw+QRUevWRfrFp45CPPkuK/zdSWcfxI10z+WVE= github.com/tikv/pd v1.1.0-beta.0.20210818082359-acba1da0018d/go.mod h1:rammPjeZgpvfrQRPkijcx8tlxF1XM5+m6kRXrkDzCAA= @@ -808,8 +810,9 @@ github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyV github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc= github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= +github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= @@ -872,8 +875,9 @@ go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b h1:3kC4J3eQF6p1UEfQTkC67eEeb3rTk+shQqdX6tFyq9Q= go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd v0.5.0-alpha.5.0.20210512015243-d19fbe541bf9 h1:MNsY1TIsWLNCMT4DzZjFOxbDKfSoULYP0OFjJ8dSxts= +go.etcd.io/etcd v0.5.0-alpha.5.0.20210512015243-d19fbe541bf9/go.mod h1:q+i20RPAmay+xq8LJ3VMOhXCNk4YCk3V7QP91meFavw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -893,8 +897,9 @@ go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhW go.uber.org/dig v1.8.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/fx v1.10.0/go.mod h1:vLRicqpG/qQEzno4SYU86iCwfT95EZza+Eba0ItuxqY= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -910,8 +915,9 @@ go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -959,7 +965,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= diff --git a/loader/lightning.go b/loader/lightning.go index 12a290efc8..a2d72614d5 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -15,6 +15,7 @@ package loader import ( "context" + "github.com/pingcap/tidb-tools/pkg/dbutil" "path/filepath" "sync" @@ -149,6 +150,7 @@ func (l *LightningLoader) restore(ctx context.Context) error { } cfg.Routes = l.cfg.RouteRules cfg.Checkpoint.Driver = lcfg.CheckpointDriverMySQL + cfg.Checkpoint.Schema = config.TiDBLightningCheckpointPrefix + dbutil.ColumnName(l.workerName) param := common.MySQLConnectParam{ Host: cfg.TiDB.Host, Port: cfg.TiDB.Port, diff --git a/loader/util.go b/loader/util.go index d6396e000a..d67d303f74 100644 --- a/loader/util.go +++ b/loader/util.go @@ -16,7 +16,6 @@ package loader import ( "crypto/sha1" "fmt" - "io/ioutil" "os" "path" "path/filepath" @@ -131,7 +130,7 @@ func getMydumpMetadata(cli *clientv3.Client, cfg *config.SubTaskConfig, workerNa return "", "", nil } - toPrint, err2 := ioutil.ReadFile(metafile) + toPrint, err2 := os.ReadFile(metafile) if err2 != nil { toPrint = []byte(err2.Error()) } From 99d948732133162bf9cdfd7a9e30c865d5dc076c Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Mon, 11 Oct 2021 16:13:18 +0800 Subject: [PATCH 20/23] support time zone Signed-off-by: Little-Wallace --- dm/config/subtask.go | 4 ++-- go.mod | 2 +- go.sum | 2 ++ loader/lightning.go | 22 +++++++++++++++++++--- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/dm/config/subtask.go b/dm/config/subtask.go index 927db70188..320128bfad 100644 --- a/dm/config/subtask.go +++ b/dm/config/subtask.go @@ -44,8 +44,8 @@ const ( DefaultShadowTableRules = "^_(.+)_(?:new|gho)$" DefaultTrashTableRules = "^_(.+)_(?:ghc|del|old)$" - ShadowTableRules = "shadow-table-rules" - TrashTableRules = "trash-table-rules" + ShadowTableRules = "shadow-table-rules" + TrashTableRules = "trash-table-rules" TiDBLightningCheckpointPrefix = "tidb_lightning_checkpoint_" ) diff --git a/go.mod b/go.mod index e091057367..9f59ec083d 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 github.com/pingcap/parser v0.0.0-20211004012448-687005894c4e - github.com/pingcap/tidb v1.1.0-beta.0.20211011021924-2f9d591ef722 + github.com/pingcap/tidb v1.1.0-beta.0.20211011071325-7e184f1d0125 github.com/pingcap/tidb-tools v5.2.0-alpha.0.20210727084616-915b22e4d42c+incompatible github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.5.1 diff --git a/go.sum b/go.sum index 3742438284..31c6b25e3d 100644 --- a/go.sum +++ b/go.sum @@ -654,6 +654,8 @@ github.com/pingcap/tidb v1.1.0-beta.0.20210914080841-89617e20b2c0/go.mod h1:MSg0 github.com/pingcap/tidb v1.1.0-beta.0.20210914112841-6ebfe8aa4257/go.mod h1:MSg0d2+iJI/d3mUOcLV82GLpw5+jNGq21Z8qcHqBI/g= github.com/pingcap/tidb v1.1.0-beta.0.20211011021924-2f9d591ef722 h1:Cb5Tk1tadtPpMu+za8L3JY/tDBe2zoY+EObwvElYVnU= github.com/pingcap/tidb v1.1.0-beta.0.20211011021924-2f9d591ef722/go.mod h1:WQHMalPfxjTrWMSSsKsPaJsM4ErDr2/ZOF4Gpd70CGQ= +github.com/pingcap/tidb v1.1.0-beta.0.20211011071325-7e184f1d0125 h1:pD5UareK8R15FQkcqxT2LMGzMjZ29jVLGVjbvd2EqCQ= +github.com/pingcap/tidb v1.1.0-beta.0.20211011071325-7e184f1d0125/go.mod h1:WQHMalPfxjTrWMSSsKsPaJsM4ErDr2/ZOF4Gpd70CGQ= github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo= github.com/pingcap/tidb-dashboard v0.0.0-20210716172320-2226872e3296/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ= github.com/pingcap/tidb-tools v5.0.3+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= diff --git a/loader/lightning.go b/loader/lightning.go index a2d72614d5..c4d6bac268 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -15,10 +15,11 @@ package loader import ( "context" - "github.com/pingcap/tidb-tools/pkg/dbutil" "path/filepath" "sync" + "github.com/pingcap/tidb-tools/pkg/dbutil" + "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/dm/pb" "github.com/pingcap/dm/dm/unit" @@ -118,7 +119,15 @@ func (l *LightningLoader) Init(ctx context.Context) (err error) { err = nil }) l.checkPoint = checkpoint - l.toDB, l.toDBConns, err = createConns(tctx, l.cfg, 1) + toCfg, err := l.cfg.Clone() + if err != nil { + return err + } + if toCfg.To.Session == nil { + toCfg.To.Session = make(map[string]string) + } + toCfg.To.Session["time_zone"] = "+00:00" + l.toDB, l.toDBConns, err = createConns(tctx, toCfg, 1) return err } @@ -150,7 +159,7 @@ func (l *LightningLoader) restore(ctx context.Context) error { } cfg.Routes = l.cfg.RouteRules cfg.Checkpoint.Driver = lcfg.CheckpointDriverMySQL - cfg.Checkpoint.Schema = config.TiDBLightningCheckpointPrefix + dbutil.ColumnName(l.workerName) + cfg.Checkpoint.Schema = config.TiDBLightningCheckpointPrefix + dbutil.TableName(l.cfg.Name, l.checkpointID()) param := common.MySQLConnectParam{ Host: cfg.TiDB.Host, Port: cfg.TiDB.Port, @@ -161,6 +170,13 @@ func (l *LightningLoader) restore(ctx context.Context) error { TLS: cfg.TiDB.TLS, } cfg.Checkpoint.DSN = param.ToDSN() + cfg.TiDB.Vars = make(map[string]string) + if l.cfg.To.Session != nil { + for k, v := range l.cfg.To.Session { + cfg.TiDB.Vars[k] = v + } + } + cfg.TiDB.Vars["time_zone"] = "+00:00" cfg.TiDB.StrSQLMode = l.cfg.LoaderConfig.SQLMode if err = cfg.Adjust(ctx); err != nil { return err From c921c46bc873f62d5e36b029dd37157aae7b8119 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Wed, 13 Oct 2021 10:33:02 +0800 Subject: [PATCH 21/23] remove checkpoint Signed-off-by: Little-Wallace --- loader/lightning.go | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/loader/lightning.go b/loader/lightning.go index c4d6bac268..46cd5c32a2 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -16,6 +16,7 @@ package loader import ( "context" "path/filepath" + "strings" "sync" "github.com/pingcap/tidb-tools/pkg/dbutil" @@ -35,6 +36,7 @@ import ( "github.com/pingcap/tidb/br/pkg/lightning" "github.com/pingcap/tidb/br/pkg/lightning/common" lcfg "github.com/pingcap/tidb/br/pkg/lightning/config" + ctl "github.com/pingcap/tidb/br/cmd/tidb-lightning-ctl" "go.etcd.io/etcd/clientv3" "go.uber.org/atomic" "go.uber.org/zap" @@ -131,6 +133,25 @@ func (l *LightningLoader) Init(ctx context.Context) (err error) { return err } +func (l *LightningLoader) runLightning(ctx context.Context, cfg *lcfg.Config) error { + l.Lock() + taskCtx, cancel := context.WithCancel(ctx) + l.cancel = cancel + l.Unlock() + err := l.core.RunOnce(taskCtx, cfg, nil) + failpoint.Inject("LoadDataSlowDownByTask", func(val failpoint.Value) { + tasks := val.(string) + taskNames := strings.Split(tasks, ",") + for _, taskName := range taskNames { + if l.cfg.Name == taskName { + l.logger.Info("inject failpoint LoadDataSlowDownByTask", zap.String("task", taskName)) + <-taskCtx.Done() + } + } + }) + return err +} + func (l *LightningLoader) restore(ctx context.Context) error { tctx := tcontext.NewContext(ctx, l.logger) if err := putLoadTask(l.cli, l.cfg, l.workerName); err != nil { @@ -160,6 +181,7 @@ func (l *LightningLoader) restore(ctx context.Context) error { cfg.Routes = l.cfg.RouteRules cfg.Checkpoint.Driver = lcfg.CheckpointDriverMySQL cfg.Checkpoint.Schema = config.TiDBLightningCheckpointPrefix + dbutil.TableName(l.cfg.Name, l.checkpointID()) + cfg.Checkpoint.KeepAfterSuccess = lcfg.CheckpointOrigin param := common.MySQLConnectParam{ Host: cfg.TiDB.Host, Port: cfg.TiDB.Port, @@ -181,13 +203,10 @@ func (l *LightningLoader) restore(ctx context.Context) error { if err = cfg.Adjust(ctx); err != nil { return err } - l.Lock() - taskCtx, cancel := context.WithCancel(ctx) - l.cancel = cancel - l.Unlock() - err = l.core.RunOnce(taskCtx, cfg, nil) + err = l.runLightning(ctx, cfg) if err == nil { l.finish.Store(true) + ctl.CheckpointRemove(ctx, cfg, "all") offsetSQL := l.checkPoint.GenSQL(lightningCheckpointFile, 1) err = l.toDBConns[0].executeSQL(tctx, []string{offsetSQL}) _ = l.checkPoint.UpdateOffset(lightningCheckpointFile, 1) From 557241d79606ba5299d00b5fbf05f551f7330585 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Tue, 26 Oct 2021 11:46:53 +0800 Subject: [PATCH 22/23] replace dumpling Signed-off-by: Little-Wallace --- CHANGELOG.md | 2 +- chaos/cases/schema.go | 4 +-- chaos/cases/stmt.go | 4 +-- chaos/cases/task.go | 2 +- dm/config/task.go | 2 +- dm/ctl/common/util.go | 2 +- dm/master/server_test.go | 6 ++--- dm/master/shardddl/optimist_test.go | 6 ++--- dm/worker/task_checker_test.go | 2 +- dumpling/dumpling.go | 2 +- dumpling/dumpling_test.go | 2 +- dumpling/metrics.go | 2 +- dumpling/util.go | 2 +- go.mod | 7 +++--- go.sum | 36 ++++++++++++++------------- loader/convert_data.go | 2 +- loader/db.go | 2 +- loader/lightning.go | 5 ++-- pkg/binlog/event/util.go | 4 +-- pkg/parser/common.go | 10 ++++---- pkg/parser/common_test.go | 2 +- pkg/retry/errors.go | 2 +- pkg/schema/tracker.go | 6 ++--- pkg/schema/tracker_test.go | 2 +- pkg/shardddl/optimism/info.go | 2 +- pkg/shardddl/optimism/info_test.go | 6 ++--- pkg/shardddl/optimism/keeper_test.go | 4 +-- pkg/shardddl/optimism/lock.go | 4 +-- pkg/shardddl/optimism/lock_test.go | 6 ++--- pkg/shardddl/optimism/schema.go | 2 +- pkg/shardddl/optimism/schema_test.go | 2 +- pkg/utils/common.go | 4 +-- pkg/utils/common_test.go | 2 +- pkg/utils/db.go | 4 +-- pkg/utils/db_test.go | 2 +- relay/common/util.go | 4 +-- relay/common/util_test.go | 2 +- relay/relay.go | 2 +- relay/relay_test.go | 2 +- relay/transformer/transformer.go | 2 +- relay/transformer/transformer_test.go | 2 +- relay/writer/file.go | 2 +- relay/writer/file_test.go | 2 +- relay/writer/file_util.go | 2 +- relay/writer/file_util_test.go | 2 +- syncer/checkpoint.go | 4 +-- syncer/dbconn/upstream_db.go | 4 +-- syncer/ddl.go | 4 +-- syncer/ddl_test.go | 4 +-- syncer/dml.go | 6 ++--- syncer/dml_test.go | 10 ++++---- syncer/error.go | 6 ++--- syncer/error_test.go | 2 +- syncer/expr_filter_group.go | 2 +- syncer/filter.go | 2 +- syncer/filter_test.go | 2 +- syncer/handle_error.go | 4 +-- syncer/online-ddl-tools/online_ddl.go | 4 +-- syncer/optimist.go | 4 +-- syncer/schema.go | 6 ++--- syncer/shardddl/optimist.go | 2 +- syncer/shardddl/optimist_test.go | 6 ++--- syncer/syncer.go | 8 +++--- syncer/syncer_test.go | 2 +- syncer/util.go | 8 +++--- syncer/util_test.go | 6 ++--- 66 files changed, 136 insertions(+), 136 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfd0187efc..f262559102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -540,7 +540,7 @@ All notable changes to this project will be documented in this file. - Remove the restriction of "the next shard DDL statement cannot be executed unless the current shard DDL operation is completely finished in shard merge scene for binlog replication" [#177](https://github.com/pingcap/dm/pull/177) - Support retry task on the `invalid connection` error for binlog replication [#66](https://github.com/pingcap/dm/pull/66) - Support generating `-schema-create.sql` files automatically for full migration [#186](https://github.com/pingcap/dm/pull/186) -- Use [TiDB SQL Parser](https://github.com/pingcap/parser) to parse and restore DDL statements in binlog query events, and remove the `use db` statement when replicating to downstream [#54](https://github.com/pingcap/dm/pull/54) +- Use [TiDB SQL Parser](https://github.com/pingcap/tidb/parser) to parse and restore DDL statements in binlog query events, and remove the `use db` statement when replicating to downstream [#54](https://github.com/pingcap/dm/pull/54) - Support migrating tables with generated column for full migration and binlog replication [#42](https://github.com/pingcap/dm/pull/42) [#60](https://github.com/pingcap/dm/pull/60) - Support appending the task name as a suffix to the dumped data directory (`dir` in the task configuration file) for full migration [#100](https://github.com/pingcap/dm/pull/100) - Support resuming tasks automatically after the DM-worker process restarted [#88](https://github.com/pingcap/dm/pull/88) [#116](https://github.com/pingcap/dm/pull/116) diff --git a/chaos/cases/schema.go b/chaos/cases/schema.go index 7028405880..2e56342c89 100644 --- a/chaos/cases/schema.go +++ b/chaos/cases/schema.go @@ -14,8 +14,8 @@ package main import ( - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" ) const ( diff --git a/chaos/cases/stmt.go b/chaos/cases/stmt.go index 07c9acc55d..5e8f27241c 100644 --- a/chaos/cases/stmt.go +++ b/chaos/cases/stmt.go @@ -17,8 +17,8 @@ import ( "math/rand" "github.com/chaos-mesh/go-sqlsmith" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" ) type dmlType int diff --git a/chaos/cases/task.go b/chaos/cases/task.go index 6fbeeba283..e2df7a9070 100644 --- a/chaos/cases/task.go +++ b/chaos/cases/task.go @@ -23,8 +23,8 @@ import ( "github.com/chaos-mesh/go-sqlsmith" "github.com/pingcap/errors" - "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-tools/pkg/dbutil" + "github.com/pingcap/tidb/parser/mysql" "go.uber.org/zap" "golang.org/x/sync/errgroup" diff --git a/dm/config/task.go b/dm/config/task.go index 76de670a96..14526a9ead 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -23,11 +23,11 @@ import ( "github.com/coreos/go-semver/semver" "github.com/dustin/go-humanize" - "github.com/pingcap/parser" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" "github.com/pingcap/tidb-tools/pkg/column-mapping" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" + "github.com/pingcap/tidb/parser" "go.uber.org/zap" "gopkg.in/yaml.v2" diff --git a/dm/ctl/common/util.go b/dm/ctl/common/util.go index bc61a9bfce..be89dd3d9b 100644 --- a/dm/ctl/common/util.go +++ b/dm/ctl/common/util.go @@ -29,8 +29,8 @@ import ( "github.com/golang/protobuf/proto" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - "github.com/pingcap/parser" toolutils "github.com/pingcap/tidb-tools/pkg/utils" + "github.com/pingcap/tidb/parser" "github.com/spf13/cobra" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" diff --git a/dm/master/server_test.go b/dm/master/server_test.go index bb6267621f..046e7eec60 100644 --- a/dm/master/server_test.go +++ b/dm/master/server_test.go @@ -32,11 +32,11 @@ import ( "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" toolutils "github.com/pingcap/tidb-tools/pkg/utils" tiddl "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" tidbmock "github.com/pingcap/tidb/util/mock" "github.com/tikv/pd/pkg/tempurl" diff --git a/dm/master/shardddl/optimist_test.go b/dm/master/shardddl/optimist_test.go index 184b012da3..3c318fe0da 100644 --- a/dm/master/shardddl/optimist_test.go +++ b/dm/master/shardddl/optimist_test.go @@ -19,12 +19,12 @@ import ( "time" . "github.com/pingcap/check" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/schemacmp" tiddl "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/util/mock" "go.etcd.io/etcd/clientv3" diff --git a/dm/worker/task_checker_test.go b/dm/worker/task_checker_test.go index cf4dabdfe9..aa60c5d8e2 100644 --- a/dm/worker/task_checker_test.go +++ b/dm/worker/task_checker_test.go @@ -18,7 +18,7 @@ import ( "github.com/pingcap/check" "github.com/pingcap/errors" - tmysql "github.com/pingcap/parser/mysql" + tmysql "github.com/pingcap/tidb/parser/mysql" "go.uber.org/zap" "github.com/pingcap/dm/dm/config" diff --git a/dumpling/dumpling.go b/dumpling/dumpling.go index 887dc2010b..3c744ab66c 100644 --- a/dumpling/dumpling.go +++ b/dumpling/dumpling.go @@ -19,7 +19,7 @@ import ( "strings" "time" - "github.com/pingcap/dumpling/v4/export" + "github.com/pingcap/tidb/dumpling/export" "github.com/pingcap/errors" "github.com/pingcap/failpoint" filter "github.com/pingcap/tidb-tools/pkg/table-filter" diff --git a/dumpling/dumpling_test.go b/dumpling/dumpling_test.go index 9f7ed03386..3aee1b0910 100644 --- a/dumpling/dumpling_test.go +++ b/dumpling/dumpling_test.go @@ -18,7 +18,7 @@ import ( "testing" "time" - "github.com/pingcap/dumpling/v4/export" + "github.com/pingcap/tidb/dumpling/export" "github.com/pingcap/failpoint" "github.com/pingcap/tidb-tools/pkg/filter" diff --git a/dumpling/metrics.go b/dumpling/metrics.go index 22bdd256cc..c471d37f8d 100644 --- a/dumpling/metrics.go +++ b/dumpling/metrics.go @@ -14,7 +14,7 @@ package dumpling import ( - "github.com/pingcap/dumpling/v4/export" + "github.com/pingcap/tidb/dumpling/export" "github.com/pingcap/failpoint" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" diff --git a/dumpling/util.go b/dumpling/util.go index aeb0174d9e..15c619d538 100644 --- a/dumpling/util.go +++ b/dumpling/util.go @@ -18,7 +18,7 @@ import ( "fmt" "strings" - "github.com/pingcap/dumpling/v4/export" + "github.com/pingcap/tidb/dumpling/export" filter "github.com/pingcap/tidb-tools/pkg/table-filter" "github.com/spf13/pflag" diff --git a/go.mod b/go.mod index 9f59ec083d..a4e4a05e19 100644 --- a/go.mod +++ b/go.mod @@ -21,13 +21,12 @@ require ( github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d github.com/labstack/echo/v4 v4.4.0 github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 - github.com/pingcap/dumpling v0.0.0-20211008101523-e13bfc29a093 github.com/pingcap/errors v0.11.5-0.20210513014640-40f9a1999b3b github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 - github.com/pingcap/parser v0.0.0-20211004012448-687005894c4e - github.com/pingcap/tidb v1.1.0-beta.0.20211011071325-7e184f1d0125 - github.com/pingcap/tidb-tools v5.2.0-alpha.0.20210727084616-915b22e4d42c+incompatible + github.com/pingcap/tidb v1.1.0-beta.0.20211026030648-c497d5c06348 + github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible + github.com/pingcap/tidb/parser v0.0.0-20211026030648-c497d5c06348 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.5.1 github.com/rakyll/statik v0.1.6 diff --git a/go.sum b/go.sum index 31c6b25e3d..4d673ba58f 100644 --- a/go.sum +++ b/go.sum @@ -215,6 +215,7 @@ github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -607,8 +608,6 @@ github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JH github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 h1:R8gStypOBmpnHEx1qi//SaqxJVI4inOqljg/Aj5/390= github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= -github.com/pingcap/dumpling v0.0.0-20211008101523-e13bfc29a093 h1:mBvWJxlVXDJ41h8U1OZmtzEUG1HFxxcWymp9M1BHg2M= -github.com/pingcap/dumpling v0.0.0-20211008101523-e13bfc29a093/go.mod h1:nfwRDefndWbtYYsjVI3jOBqyH3PgHRX0hdshNicMy5o= github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg= github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -632,8 +631,10 @@ github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20210219064844-c1844a4775d6/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20210805052247-76981389e818/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4 h1:4EUpHzPFHwleKkVALyMqQbQcNziPZvU+vhUT9Wzj93E= github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20210915062418-0f5764a128ad/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20211011042309-a4518fcacbc8 h1:IBNpJoFfn8BHhqyeKuAuVnYpViQE6/LOXFtDxuQPbqA= +github.com/pingcap/kvproto v0.0.0-20211011042309-a4518fcacbc8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= @@ -643,26 +644,28 @@ github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 h1:SvWCbCPh1YeHd9yQLks github.com/pingcap/log v0.0.0-20210906054005-afc726e70354/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/parser v0.0.0-20210415081931-48e7f467fd74/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= +github.com/pingcap/parser v0.0.0-20210907051057-948434fa20e4 h1:v3paTSRJgM7E/2CqIxKd5gNSrlknktjIFNS9NMbpFLo= github.com/pingcap/parser v0.0.0-20210907051057-948434fa20e4/go.mod h1:Ek0mLKEqUGnQqBw1JnYrJQxsguU433DU68yUbsoeJ7s= -github.com/pingcap/parser v0.0.0-20211004012448-687005894c4e h1:dPMDpj+7ng9qEWoT3n6qjpB1ohz79uTLVM6ILW+ZMT0= -github.com/pingcap/parser v0.0.0-20211004012448-687005894c4e/go.mod h1:+xcMiiZzdIktT/Nqdfm81dkECJ2EPuoAYywd57py4Pk= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3/go.mod h1:tckvA041UWP+NqYzrJ3fMgC/Hw9wnmQ/tUkp/JaHly8= github.com/pingcap/sysutil v0.0.0-20210730114356-fcd8a63f68c5 h1:7rvAtZe/ZUzOKzgriNPQoBNvleJXBk4z7L3Z47+tS98= github.com/pingcap/sysutil v0.0.0-20210730114356-fcd8a63f68c5/go.mod h1:XsOaV712rUk63aOEKYP9PhXTIE3FMNHmC2r1wX5wElY= github.com/pingcap/tidb v1.1.0-beta.0.20210914080841-89617e20b2c0/go.mod h1:MSg0d2+iJI/d3mUOcLV82GLpw5+jNGq21Z8qcHqBI/g= -github.com/pingcap/tidb v1.1.0-beta.0.20210914112841-6ebfe8aa4257/go.mod h1:MSg0d2+iJI/d3mUOcLV82GLpw5+jNGq21Z8qcHqBI/g= -github.com/pingcap/tidb v1.1.0-beta.0.20211011021924-2f9d591ef722 h1:Cb5Tk1tadtPpMu+za8L3JY/tDBe2zoY+EObwvElYVnU= -github.com/pingcap/tidb v1.1.0-beta.0.20211011021924-2f9d591ef722/go.mod h1:WQHMalPfxjTrWMSSsKsPaJsM4ErDr2/ZOF4Gpd70CGQ= -github.com/pingcap/tidb v1.1.0-beta.0.20211011071325-7e184f1d0125 h1:pD5UareK8R15FQkcqxT2LMGzMjZ29jVLGVjbvd2EqCQ= -github.com/pingcap/tidb v1.1.0-beta.0.20211011071325-7e184f1d0125/go.mod h1:WQHMalPfxjTrWMSSsKsPaJsM4ErDr2/ZOF4Gpd70CGQ= +github.com/pingcap/tidb v1.1.0-beta.0.20211026030648-c497d5c06348 h1:DVcIszrlW/71/VA1Vpu0Sl6kY1oJiGRRHEpP+LqNrw0= +github.com/pingcap/tidb v1.1.0-beta.0.20211026030648-c497d5c06348/go.mod h1:0x9V4RDM9rVr1FduK7jZLG3E6gp4ggqB896y1l2HFUI= github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo= github.com/pingcap/tidb-dashboard v0.0.0-20210716172320-2226872e3296/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ= github.com/pingcap/tidb-tools v5.0.3+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tidb-tools v5.2.0-alpha.0.20210727084616-915b22e4d42c+incompatible h1:3UusiiE+cbOYzjMT3oxk2gN3lUT0ccjMiul51EEvZZc= -github.com/pingcap/tidb-tools v5.2.0-alpha.0.20210727084616-915b22e4d42c+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20210802080519-94b831c6db55 h1:oxOovwOzm7VD37XpDo9NUtfGddZMwLpjtaQOxAq6HKg= +github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible h1:c7+izmker91NkjkZ6FgTlmD4k1A5FLOAq+li6Ki2/GY= +github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg= +github.com/pingcap/tidb/parser v0.0.0-20211012113926-e933e0008d04 h1:NQGT/24Cco2ypvBb5W/Eg1PEyKaL3TpGhrGD3VlfwDQ= +github.com/pingcap/tidb/parser v0.0.0-20211012113926-e933e0008d04/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg= +github.com/pingcap/tidb/parser v0.0.0-20211026030648-c497d5c06348 h1:Dt7Lsp9i6pMF/eKp/wMG2lO9ZsTM/gLSwRfeLUI3SDk= +github.com/pingcap/tidb/parser v0.0.0-20211026030648-c497d5c06348/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg= github.com/pingcap/tipb v0.0.0-20210802080519-94b831c6db55/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pingcap/tipb v0.0.0-20211008080435-3fd327dfce0e h1:fZY5T65QWiPc9noQJ1UkdwejZyBZjNfxzSyTcBjKrEU= +github.com/pingcap/tipb v0.0.0-20211008080435-3fd327dfce0e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -802,8 +805,8 @@ github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ= github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210913094925-a8fa8acd44e7/go.mod h1:KwtZXt0JD+bP9bWW2ka0ir3Wp3oTEfZUTh22bs2sI4o= -github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210926100628-3cc2459779ca h1:aE/y7lkygl716M6eIos95BKjM1mGew+EoLWRQAXDoAE= -github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210926100628-3cc2459779ca/go.mod h1:KwtZXt0JD+bP9bWW2ka0ir3Wp3oTEfZUTh22bs2sI4o= +github.com/tikv/client-go/v2 v2.0.0-alpha.0.20211011083157-49c8dd23f1f0 h1:DZdVqFVe+r7M0tSZ2LdqeY+UNplR1DSLCdGtH1AKi5M= +github.com/tikv/client-go/v2 v2.0.0-alpha.0.20211011083157-49c8dd23f1f0/go.mod h1:00plYwQsQ5kBUmafHO+JkjznGgFaBokMZl82TZIbsQk= github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d/go.mod h1:Jw9KG11C/23Rr7DW4XWQ7H5xOgGZo6DFL1OKAF4+Igw= github.com/tikv/pd v1.1.0-beta.0.20210818082359-acba1da0018d h1:AFm1Dzw+QRUevWRfrFp45CPPkuK/zdSWcfxI10z+WVE= github.com/tikv/pd v1.1.0-beta.0.20210818082359-acba1da0018d/go.mod h1:rammPjeZgpvfrQRPkijcx8tlxF1XM5+m6kRXrkDzCAA= @@ -854,9 +857,8 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= +github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457 h1:tBbuFCtyJNKT+BFAv6qjvTFpVdy97IYNaBwGUXifIUs= github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8= -github.com/xitongsys/parquet-go v1.6.0 h1:j6YrTVZdQx5yywJLIOklZcKVsCoSD1tqOVRXyTBFSjs= -github.com/xitongsys/parquet-go v1.6.0/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8= github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= diff --git a/loader/convert_data.go b/loader/convert_data.go index 9d3e899f1b..5013d65200 100644 --- a/loader/convert_data.go +++ b/loader/convert_data.go @@ -28,9 +28,9 @@ import ( "github.com/pingcap/dm/pkg/utils" "github.com/pingcap/errors" - "github.com/pingcap/parser/ast" cm "github.com/pingcap/tidb-tools/pkg/column-mapping" router "github.com/pingcap/tidb-tools/pkg/table-router" + "github.com/pingcap/tidb/parser/ast" ) func bytes2str(bs []byte) string { diff --git a/loader/db.go b/loader/db.go index d6de85f760..a581386225 100644 --- a/loader/db.go +++ b/loader/db.go @@ -21,8 +21,8 @@ import ( "github.com/go-sql-driver/mysql" "github.com/pingcap/failpoint" - tmysql "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-tools/pkg/dbutil" + tmysql "github.com/pingcap/tidb/parser/mysql" "go.uber.org/zap" "github.com/pingcap/dm/dm/config" diff --git a/loader/lightning.go b/loader/lightning.go index 46cd5c32a2..f4780ff261 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -32,11 +32,10 @@ import ( "github.com/docker/go-units" "github.com/pingcap/failpoint" - "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/br/pkg/lightning" "github.com/pingcap/tidb/br/pkg/lightning/common" lcfg "github.com/pingcap/tidb/br/pkg/lightning/config" - ctl "github.com/pingcap/tidb/br/cmd/tidb-lightning-ctl" + "github.com/pingcap/tidb/parser/mysql" "go.etcd.io/etcd/clientv3" "go.uber.org/atomic" "go.uber.org/zap" @@ -206,7 +205,7 @@ func (l *LightningLoader) restore(ctx context.Context) error { err = l.runLightning(ctx, cfg) if err == nil { l.finish.Store(true) - ctl.CheckpointRemove(ctx, cfg, "all") + lightning.CheckpointRemove(ctx, cfg, "all") offsetSQL := l.checkPoint.GenSQL(lightningCheckpointFile, 1) err = l.toDBConns[0].executeSQL(tctx, []string{offsetSQL}) _ = l.checkPoint.UpdateOffset(lightningCheckpointFile, 1) diff --git a/pkg/binlog/event/util.go b/pkg/binlog/event/util.go index acffde25e1..b205fb765d 100644 --- a/pkg/binlog/event/util.go +++ b/pkg/binlog/event/util.go @@ -28,8 +28,8 @@ import ( gmysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" - "github.com/pingcap/parser" - "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/dm/pkg/terror" ) diff --git a/pkg/parser/common.go b/pkg/parser/common.go index 5b201b1d75..e27f8ec19c 100644 --- a/pkg/parser/common.go +++ b/pkg/parser/common.go @@ -20,18 +20,18 @@ import ( "github.com/pingcap/dm/pkg/terror" "github.com/pingcap/dm/pkg/utils" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/format" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/format" + "github.com/pingcap/tidb/parser/model" _ "github.com/pingcap/tidb/types/parser_driver" // for import parser driver "go.uber.org/zap" ) const ( // SingleRenameTableNameNum stands for number of TableNames in a single table renaming. it's 2 after - // https://github.com/pingcap/parser/pull/1021 + // https://github.com/pingcap/tidb/parser/pull/1021 SingleRenameTableNameNum = 2 ) diff --git a/pkg/parser/common_test.go b/pkg/parser/common_test.go index 2fcd4d2133..f0adc5b384 100644 --- a/pkg/parser/common_test.go +++ b/pkg/parser/common_test.go @@ -17,8 +17,8 @@ import ( "testing" . "github.com/pingcap/check" - "github.com/pingcap/parser" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser" "github.com/pingcap/dm/pkg/terror" "github.com/pingcap/dm/pkg/utils" diff --git a/pkg/retry/errors.go b/pkg/retry/errors.go index ed70343e2b..1c0e2f00a5 100644 --- a/pkg/retry/errors.go +++ b/pkg/retry/errors.go @@ -18,7 +18,7 @@ import ( gmysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/pingcap/errors" - tmysql "github.com/pingcap/parser/mysql" + tmysql "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/dm/pkg/terror" ) diff --git a/pkg/schema/tracker.go b/pkg/schema/tracker.go index 50ebc06a1b..cd204256fa 100644 --- a/pkg/schema/tracker.go +++ b/pkg/schema/tracker.go @@ -19,15 +19,15 @@ import ( "strings" "github.com/pingcap/errors" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" - "github.com/pingcap/parser/terror" "github.com/pingcap/tidb-tools/pkg/filter" tidbConfig "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" + "github.com/pingcap/tidb/parser/terror" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/mockstore" diff --git a/pkg/schema/tracker_test.go b/pkg/schema/tracker_test.go index cc9345511d..4253942197 100644 --- a/pkg/schema/tracker_test.go +++ b/pkg/schema/tracker_test.go @@ -25,8 +25,8 @@ import ( "github.com/DATA-DOG/go-sqlmock" . "github.com/pingcap/check" "github.com/pingcap/log" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser/model" "go.uber.org/zap/zapcore" "github.com/pingcap/dm/pkg/conn" diff --git a/pkg/shardddl/optimism/info.go b/pkg/shardddl/optimism/info.go index d964d810b6..13cd821066 100644 --- a/pkg/shardddl/optimism/info.go +++ b/pkg/shardddl/optimism/info.go @@ -18,8 +18,8 @@ import ( "encoding/json" "fmt" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/schemacmp" + "github.com/pingcap/tidb/parser/model" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/mvcc/mvccpb" diff --git a/pkg/shardddl/optimism/info_test.go b/pkg/shardddl/optimism/info_test.go index 1471d0d600..c85a9d10e7 100644 --- a/pkg/shardddl/optimism/info_test.go +++ b/pkg/shardddl/optimism/info_test.go @@ -21,10 +21,10 @@ import ( "time" . "github.com/pingcap/check" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" tiddl "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/util/mock" "go.etcd.io/etcd/clientv3" diff --git a/pkg/shardddl/optimism/keeper_test.go b/pkg/shardddl/optimism/keeper_test.go index 403a6b7223..a6fb82522f 100644 --- a/pkg/shardddl/optimism/keeper_test.go +++ b/pkg/shardddl/optimism/keeper_test.go @@ -17,9 +17,9 @@ import ( "testing" . "github.com/pingcap/check" - "github.com/pingcap/parser" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/schemacmp" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/util/mock" "go.etcd.io/etcd/integration" diff --git a/pkg/shardddl/optimism/lock.go b/pkg/shardddl/optimism/lock.go index 683de1bc81..072a68a6ae 100644 --- a/pkg/shardddl/optimism/lock.go +++ b/pkg/shardddl/optimism/lock.go @@ -17,9 +17,9 @@ import ( "fmt" "sync" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" "github.com/pingcap/tidb-tools/pkg/schemacmp" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" diff --git a/pkg/shardddl/optimism/lock_test.go b/pkg/shardddl/optimism/lock_test.go index a231344175..3c1502c2d8 100644 --- a/pkg/shardddl/optimism/lock_test.go +++ b/pkg/shardddl/optimism/lock_test.go @@ -17,10 +17,10 @@ import ( "testing" . "github.com/pingcap/check" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/schemacmp" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/util/mock" "go.etcd.io/etcd/integration" diff --git a/pkg/shardddl/optimism/schema.go b/pkg/shardddl/optimism/schema.go index 2d81368718..8673b8cb40 100644 --- a/pkg/shardddl/optimism/schema.go +++ b/pkg/shardddl/optimism/schema.go @@ -16,7 +16,7 @@ package optimism import ( "encoding/json" - "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/parser/model" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/clientv3/clientv3util" diff --git a/pkg/shardddl/optimism/schema_test.go b/pkg/shardddl/optimism/schema_test.go index 1fc7a97d2c..f5f3ba582a 100644 --- a/pkg/shardddl/optimism/schema_test.go +++ b/pkg/shardddl/optimism/schema_test.go @@ -15,7 +15,7 @@ package optimism import ( . "github.com/pingcap/check" - "github.com/pingcap/parser" + "github.com/pingcap/tidb/parser" "github.com/pingcap/tidb/util/mock" ) diff --git a/pkg/utils/common.go b/pkg/utils/common.go index 093deaaf3e..ada5b0eb51 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -23,11 +23,11 @@ import ( "time" "github.com/pingcap/failpoint" - "github.com/pingcap/parser/model" - tmysql "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" + "github.com/pingcap/tidb/parser/model" + tmysql "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/table" diff --git a/pkg/utils/common_test.go b/pkg/utils/common_test.go index 5b2b8de634..7178733651 100644 --- a/pkg/utils/common_test.go +++ b/pkg/utils/common_test.go @@ -20,9 +20,9 @@ import ( "github.com/DATA-DOG/go-sqlmock" . "github.com/pingcap/check" - "github.com/pingcap/parser" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" + "github.com/pingcap/tidb/parser" ) var _ = Suite(&testCommonSuite{}) diff --git a/pkg/utils/db.go b/pkg/utils/db.go index 948126c972..80fede3aee 100644 --- a/pkg/utils/db.go +++ b/pkg/utils/db.go @@ -28,10 +28,10 @@ import ( "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - "github.com/pingcap/parser" - tmysql "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-tools/pkg/check" "github.com/pingcap/tidb-tools/pkg/dbutil" + "github.com/pingcap/tidb/parser" + tmysql "github.com/pingcap/tidb/parser/mysql" "go.uber.org/zap" "github.com/pingcap/dm/pkg/gtid" diff --git a/pkg/utils/db_test.go b/pkg/utils/db_test.go index 2dbd7fa145..bb67465080 100644 --- a/pkg/utils/db_test.go +++ b/pkg/utils/db_test.go @@ -24,7 +24,7 @@ import ( "github.com/go-sql-driver/mysql" . "github.com/pingcap/check" "github.com/pingcap/errors" - tmysql "github.com/pingcap/parser/mysql" + tmysql "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/dm/pkg/gtid" ) diff --git a/relay/common/util.go b/relay/common/util.go index 241c0f6ad2..bbbe0886e0 100644 --- a/relay/common/util.go +++ b/relay/common/util.go @@ -14,8 +14,8 @@ package common import ( - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/dm/pkg/utils" diff --git a/relay/common/util_test.go b/relay/common/util_test.go index a4e11aa1c6..d0e23a211b 100644 --- a/relay/common/util_test.go +++ b/relay/common/util_test.go @@ -17,7 +17,7 @@ import ( "testing" "github.com/pingcap/check" - "github.com/pingcap/parser" + "github.com/pingcap/tidb/parser" ) func TestSuite(t *testing.T) { diff --git a/relay/relay.go b/relay/relay.go index 12cdf3a62f..e7e7fbfc7c 100644 --- a/relay/relay.go +++ b/relay/relay.go @@ -27,8 +27,8 @@ import ( "github.com/go-mysql-org/go-mysql/replication" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - "github.com/pingcap/parser" toolutils "github.com/pingcap/tidb-tools/pkg/utils" + "github.com/pingcap/tidb/parser" "go.uber.org/atomic" "go.uber.org/zap" diff --git a/relay/relay_test.go b/relay/relay_test.go index b58eee9935..28ad060d97 100644 --- a/relay/relay_test.go +++ b/relay/relay_test.go @@ -29,7 +29,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - "github.com/pingcap/parser" + "github.com/pingcap/tidb/parser" "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/pkg/binlog/event" diff --git a/relay/transformer/transformer.go b/relay/transformer/transformer.go index 4811750eba..974e3b0c4a 100644 --- a/relay/transformer/transformer.go +++ b/relay/transformer/transformer.go @@ -16,7 +16,7 @@ package transformer import ( "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" - "github.com/pingcap/parser" + "github.com/pingcap/tidb/parser" "github.com/pingcap/dm/relay/common" ) diff --git a/relay/transformer/transformer_test.go b/relay/transformer/transformer_test.go index 216dcf78ed..97d7cd50e8 100644 --- a/relay/transformer/transformer_test.go +++ b/relay/transformer/transformer_test.go @@ -20,7 +20,7 @@ import ( "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" "github.com/pingcap/check" - "github.com/pingcap/parser" + "github.com/pingcap/tidb/parser" "github.com/pingcap/dm/pkg/binlog/event" "github.com/pingcap/dm/pkg/gtid" diff --git a/relay/writer/file.go b/relay/writer/file.go index bdc69007ab..658dbcfe48 100644 --- a/relay/writer/file.go +++ b/relay/writer/file.go @@ -23,7 +23,7 @@ import ( gmysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" "github.com/pingcap/failpoint" - "github.com/pingcap/parser" + "github.com/pingcap/tidb/parser" "go.uber.org/atomic" "go.uber.org/zap" diff --git a/relay/writer/file_test.go b/relay/writer/file_test.go index 3a09f23b1a..0c959a99bb 100644 --- a/relay/writer/file_test.go +++ b/relay/writer/file_test.go @@ -25,7 +25,7 @@ import ( gmysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" "github.com/pingcap/check" - "github.com/pingcap/parser" + "github.com/pingcap/tidb/parser" "github.com/pingcap/dm/pkg/binlog/common" "github.com/pingcap/dm/pkg/binlog/event" diff --git a/relay/writer/file_util.go b/relay/writer/file_util.go index dfd76808cf..937dfc8587 100644 --- a/relay/writer/file_util.go +++ b/relay/writer/file_util.go @@ -24,7 +24,7 @@ import ( gmysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" "github.com/google/uuid" - "github.com/pingcap/parser" + "github.com/pingcap/tidb/parser" "github.com/pingcap/dm/pkg/binlog/event" "github.com/pingcap/dm/pkg/binlog/reader" diff --git a/relay/writer/file_util_test.go b/relay/writer/file_util_test.go index 8443ced7d0..add3fe95e0 100644 --- a/relay/writer/file_util_test.go +++ b/relay/writer/file_util_test.go @@ -25,7 +25,7 @@ import ( "github.com/go-mysql-org/go-mysql/replication" "github.com/pingcap/check" "github.com/pingcap/errors" - "github.com/pingcap/parser" + "github.com/pingcap/tidb/parser" "github.com/pingcap/dm/pkg/binlog/event" "github.com/pingcap/dm/pkg/gtid" diff --git a/syncer/checkpoint.go b/syncer/checkpoint.go index fc1edd3ca2..a57d277227 100644 --- a/syncer/checkpoint.go +++ b/syncer/checkpoint.go @@ -39,10 +39,10 @@ import ( "github.com/go-mysql-org/go-mysql/mysql" "github.com/pingcap/failpoint" - "github.com/pingcap/parser/model" - tmysql "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser/model" + tmysql "github.com/pingcap/tidb/parser/mysql" "go.uber.org/zap" ) diff --git a/syncer/dbconn/upstream_db.go b/syncer/dbconn/upstream_db.go index 982815ca64..d9609ab774 100644 --- a/syncer/dbconn/upstream_db.go +++ b/syncer/dbconn/upstream_db.go @@ -18,9 +18,9 @@ import ( "github.com/go-mysql-org/go-mysql/mysql" "github.com/pingcap/failpoint" - "github.com/pingcap/parser" - tmysql "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser" + tmysql "github.com/pingcap/tidb/parser/mysql" "go.uber.org/zap" "github.com/pingcap/dm/dm/config" diff --git a/syncer/ddl.go b/syncer/ddl.go index 43fdf30fe7..0971933e43 100644 --- a/syncer/ddl.go +++ b/syncer/ddl.go @@ -17,9 +17,9 @@ import ( "time" "github.com/go-mysql-org/go-mysql/replication" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" "go.uber.org/zap" tcontext "github.com/pingcap/dm/pkg/context" diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index bc163f5d3a..1468ca16dc 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -23,11 +23,11 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/go-sql-driver/mysql" . "github.com/pingcap/check" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" "github.com/pingcap/tidb/br/pkg/mock" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" "go.uber.org/zap" "github.com/pingcap/dm/dm/config" diff --git a/syncer/dml.go b/syncer/dml.go index ece76443de..34d732a4e5 100644 --- a/syncer/dml.go +++ b/syncer/dml.go @@ -19,11 +19,11 @@ import ( "strconv" "strings" - "github.com/pingcap/parser/model" - "github.com/pingcap/parser/mysql" - "github.com/pingcap/parser/types" "github.com/pingcap/tidb-tools/pkg/filter" "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/parser/model" + "github.com/pingcap/tidb/parser/mysql" + "github.com/pingcap/tidb/parser/types" "go.uber.org/zap" "github.com/pingcap/dm/pkg/log" diff --git a/syncer/dml_test.go b/syncer/dml_test.go index f4dbb313ab..3126212ac0 100644 --- a/syncer/dml_test.go +++ b/syncer/dml_test.go @@ -17,12 +17,12 @@ import ( "math" . "github.com/pingcap/check" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" - "github.com/pingcap/parser/mysql" - "github.com/pingcap/parser/types" tiddl "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" + "github.com/pingcap/tidb/parser/mysql" + "github.com/pingcap/tidb/parser/types" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/util/mock" ) diff --git a/syncer/error.go b/syncer/error.go index ba8d732127..2af0cbdcd3 100644 --- a/syncer/error.go +++ b/syncer/error.go @@ -20,13 +20,13 @@ import ( "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" - tmysql "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-tools/pkg/dbutil" tddl "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + tmysql "github.com/pingcap/tidb/parser/mysql" "go.uber.org/zap" tcontext "github.com/pingcap/dm/pkg/context" diff --git a/syncer/error_test.go b/syncer/error_test.go index 70551cdd5f..beee35e233 100644 --- a/syncer/error_test.go +++ b/syncer/error_test.go @@ -20,9 +20,9 @@ import ( "github.com/go-sql-driver/mysql" . "github.com/pingcap/check" "github.com/pingcap/errors" - tmysql "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/infoschema" + tmysql "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/dm/pkg/conn" tcontext "github.com/pingcap/dm/pkg/context" diff --git a/syncer/expr_filter_group.go b/syncer/expr_filter_group.go index c6eae4dd8f..7d77a5d983 100644 --- a/syncer/expr_filter_group.go +++ b/syncer/expr_filter_group.go @@ -1,10 +1,10 @@ package syncer import ( - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/filter" "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/util/chunk" "go.uber.org/zap" diff --git a/syncer/filter.go b/syncer/filter.go index 36261280f2..5dc5bdf217 100644 --- a/syncer/filter.go +++ b/syncer/filter.go @@ -15,9 +15,9 @@ package syncer import ( "github.com/go-mysql-org/go-mysql/replication" - "github.com/pingcap/parser/ast" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/dm/pkg/terror" "github.com/pingcap/dm/pkg/utils" diff --git a/syncer/filter_test.go b/syncer/filter_test.go index 58ad3ce12f..bad1d983dc 100644 --- a/syncer/filter_test.go +++ b/syncer/filter_test.go @@ -20,9 +20,9 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/go-mysql-org/go-mysql/replication" . "github.com/pingcap/check" - "github.com/pingcap/parser" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser" "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/pkg/conn" diff --git a/syncer/handle_error.go b/syncer/handle_error.go index 64613187bf..415bb9c554 100644 --- a/syncer/handle_error.go +++ b/syncer/handle_error.go @@ -17,10 +17,10 @@ import ( "context" "fmt" - "github.com/pingcap/parser" + "github.com/pingcap/tidb/parser" "github.com/go-mysql-org/go-mysql/replication" - "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/dm/dm/pb" "github.com/pingcap/dm/pkg/binlog" diff --git a/syncer/online-ddl-tools/online_ddl.go b/syncer/online-ddl-tools/online_ddl.go index a824ed2848..3f6e2d6673 100644 --- a/syncer/online-ddl-tools/online_ddl.go +++ b/syncer/online-ddl-tools/online_ddl.go @@ -29,10 +29,10 @@ import ( "github.com/pingcap/dm/syncer/dbconn" "github.com/pingcap/failpoint" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" "go.uber.org/zap" ) diff --git a/syncer/optimist.go b/syncer/optimist.go index 9d21785089..fdeab2bf59 100644 --- a/syncer/optimist.go +++ b/syncer/optimist.go @@ -17,9 +17,9 @@ import ( "context" "github.com/pingcap/failpoint" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" "go.uber.org/zap" "github.com/pingcap/dm/pkg/log" diff --git a/syncer/schema.go b/syncer/schema.go index 85fba54371..113aa1ce65 100644 --- a/syncer/schema.go +++ b/syncer/schema.go @@ -17,10 +17,10 @@ import ( "context" "strings" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/format" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/format" + "github.com/pingcap/tidb/parser/model" "go.uber.org/zap" "github.com/pingcap/dm/dm/config" diff --git a/syncer/shardddl/optimist.go b/syncer/shardddl/optimist.go index 01cea5788c..8d129198e4 100644 --- a/syncer/shardddl/optimist.go +++ b/syncer/shardddl/optimist.go @@ -17,7 +17,7 @@ import ( "context" "sync" - "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/parser/model" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" diff --git a/syncer/shardddl/optimist_test.go b/syncer/shardddl/optimist_test.go index 8e3c9bc24c..5f4017be0c 100644 --- a/syncer/shardddl/optimist_test.go +++ b/syncer/shardddl/optimist_test.go @@ -18,10 +18,10 @@ import ( "fmt" . "github.com/pingcap/check" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" tiddl "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/util/mock" diff --git a/syncer/syncer.go b/syncer/syncer.go index 0d1835f742..fd6db1852e 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -30,16 +30,16 @@ import ( "github.com/go-mysql-org/go-mysql/replication" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/format" - "github.com/pingcap/parser/model" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" cm "github.com/pingcap/tidb-tools/pkg/column-mapping" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" toolutils "github.com/pingcap/tidb-tools/pkg/utils" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/format" + "github.com/pingcap/tidb/parser/model" "go.etcd.io/etcd/clientv3" "go.uber.org/atomic" "go.uber.org/zap" diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index 7957170176..77b51e63f0 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -49,11 +49,11 @@ import ( "github.com/go-mysql-org/go-mysql/replication" _ "github.com/go-sql-driver/mysql" . "github.com/pingcap/check" - "github.com/pingcap/parser" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" cm "github.com/pingcap/tidb-tools/pkg/column-mapping" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" + "github.com/pingcap/tidb/parser" "go.uber.org/zap" ) diff --git a/syncer/util.go b/syncer/util.go index 1bc4bcc33c..93f9ce78ca 100644 --- a/syncer/util.go +++ b/syncer/util.go @@ -16,11 +16,11 @@ package syncer import ( "fmt" - dcontext "github.com/pingcap/dumpling/v4/context" - "github.com/pingcap/dumpling/v4/export" - dlog "github.com/pingcap/dumpling/v4/log" - "github.com/pingcap/parser/ast" + dcontext "github.com/pingcap/tidb/dumpling/context" + "github.com/pingcap/tidb/dumpling/export" + dlog "github.com/pingcap/tidb/dumpling/log" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser/ast" "go.uber.org/zap" "github.com/pingcap/dm/pkg/conn" diff --git a/syncer/util_test.go b/syncer/util_test.go index ac6a481089..b2f1dc1264 100644 --- a/syncer/util_test.go +++ b/syncer/util_test.go @@ -15,10 +15,10 @@ package syncer import ( . "github.com/pingcap/check" - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" _ "github.com/pingcap/tidb/types/parser_driver" ) From ad92b60625077e1ae2b4bf6291721f6f7441269b Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Tue, 26 Oct 2021 12:03:27 +0800 Subject: [PATCH 23/23] revert lightning Signed-off-by: Little-Wallace --- dm/config/task.go | 1 - dumpling/dumpling.go | 2 +- dumpling/dumpling_test.go | 2 +- dumpling/metrics.go | 2 +- dumpling/util.go | 2 +- loader/lightning.go | 67 ++++++++++----------------------------- syncer/util.go | 2 +- 7 files changed, 21 insertions(+), 57 deletions(-) diff --git a/dm/config/task.go b/dm/config/task.go index 48e2ab6a1d..435398ae6d 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -709,7 +709,6 @@ func (c *TaskConfig) adjust() error { return nil } - // getGenerateName generates name by rule or gets name from nameMap // if it's a new name, increase nameIdx // otherwise return current nameIdx. diff --git a/dumpling/dumpling.go b/dumpling/dumpling.go index 3c744ab66c..7bb837525b 100644 --- a/dumpling/dumpling.go +++ b/dumpling/dumpling.go @@ -19,10 +19,10 @@ import ( "strings" "time" - "github.com/pingcap/tidb/dumpling/export" "github.com/pingcap/errors" "github.com/pingcap/failpoint" filter "github.com/pingcap/tidb-tools/pkg/table-filter" + "github.com/pingcap/tidb/dumpling/export" "github.com/prometheus/client_golang/prometheus" "go.uber.org/atomic" "go.uber.org/zap" diff --git a/dumpling/dumpling_test.go b/dumpling/dumpling_test.go index 3aee1b0910..81fd7cf5d9 100644 --- a/dumpling/dumpling_test.go +++ b/dumpling/dumpling_test.go @@ -18,9 +18,9 @@ import ( "testing" "time" - "github.com/pingcap/tidb/dumpling/export" "github.com/pingcap/failpoint" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/dumpling/export" "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/dm/pb" diff --git a/dumpling/metrics.go b/dumpling/metrics.go index c471d37f8d..b866f76a06 100644 --- a/dumpling/metrics.go +++ b/dumpling/metrics.go @@ -14,8 +14,8 @@ package dumpling import ( - "github.com/pingcap/tidb/dumpling/export" "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/dumpling/export" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" diff --git a/dumpling/util.go b/dumpling/util.go index 15c619d538..170e0617a8 100644 --- a/dumpling/util.go +++ b/dumpling/util.go @@ -18,8 +18,8 @@ import ( "fmt" "strings" - "github.com/pingcap/tidb/dumpling/export" filter "github.com/pingcap/tidb-tools/pkg/table-filter" + "github.com/pingcap/tidb/dumpling/export" "github.com/spf13/pflag" dutils "github.com/pingcap/dm/pkg/dumpling" diff --git a/loader/lightning.go b/loader/lightning.go index 1d807188ee..27edc3a593 100644 --- a/loader/lightning.go +++ b/loader/lightning.go @@ -15,22 +15,12 @@ package loader import ( "context" - "github.com/pingcap/tidb-tools/pkg/dbutil" "path/filepath" - "strings" "sync" - "github.com/pingcap/dm/dm/config" - "github.com/pingcap/dm/dm/pb" - "github.com/pingcap/dm/dm/unit" - "github.com/pingcap/dm/pkg/binlog" - "github.com/pingcap/dm/pkg/conn" - tcontext "github.com/pingcap/dm/pkg/context" - "github.com/pingcap/dm/pkg/log" - "github.com/pingcap/dm/pkg/utils" - "github.com/docker/go-units" "github.com/pingcap/failpoint" + "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb/br/pkg/lightning" "github.com/pingcap/tidb/br/pkg/lightning/common" lcfg "github.com/pingcap/tidb/br/pkg/lightning/config" @@ -38,6 +28,15 @@ import ( "go.etcd.io/etcd/clientv3" "go.uber.org/atomic" "go.uber.org/zap" + + "github.com/pingcap/dm/dm/config" + "github.com/pingcap/dm/dm/pb" + "github.com/pingcap/dm/dm/unit" + "github.com/pingcap/dm/pkg/binlog" + "github.com/pingcap/dm/pkg/conn" + tcontext "github.com/pingcap/dm/pkg/context" + "github.com/pingcap/dm/pkg/log" + "github.com/pingcap/dm/pkg/utils" ) const ( @@ -119,34 +118,7 @@ func (l *LightningLoader) Init(ctx context.Context) (err error) { err = nil }) l.checkPoint = checkpoint - toCfg, err := l.cfg.Clone() - if err != nil { - return err - } - if toCfg.To.Session == nil { - toCfg.To.Session = make(map[string]string) - } - toCfg.To.Session["time_zone"] = "+00:00" - l.toDB, l.toDBConns, err = createConns(tctx, toCfg, 1) - return err -} - -func (l *LightningLoader) runLightning(ctx context.Context, cfg *lcfg.Config) error { - l.Lock() - taskCtx, cancel := context.WithCancel(ctx) - l.cancel = cancel - l.Unlock() - err := l.core.RunOnce(taskCtx, cfg, nil) - failpoint.Inject("LoadDataSlowDownByTask", func(val failpoint.Value) { - tasks := val.(string) - taskNames := strings.Split(tasks, ",") - for _, taskName := range taskNames { - if l.cfg.Name == taskName { - l.logger.Info("inject failpoint LoadDataSlowDownByTask", zap.String("task", taskName)) - <-taskCtx.Done() - } - } - }) + l.toDB, l.toDBConns, err = createConns(tctx, l.cfg, 1) return err } @@ -178,9 +150,7 @@ func (l *LightningLoader) restore(ctx context.Context) error { } cfg.Routes = l.cfg.RouteRules cfg.Checkpoint.Driver = lcfg.CheckpointDriverMySQL - //cfg.Checkpoint.Schema = config.TiDBLightningCheckpointPrefix + dbutil.TableName(l.cfg.Name, l.checkpointID()) cfg.Checkpoint.Schema = config.TiDBLightningCheckpointPrefix + dbutil.ColumnName(l.workerName) - cfg.Checkpoint.KeepAfterSuccess = lcfg.CheckpointOrigin param := common.MySQLConnectParam{ Host: cfg.TiDB.Host, Port: cfg.TiDB.Port, @@ -191,22 +161,17 @@ func (l *LightningLoader) restore(ctx context.Context) error { TLS: cfg.TiDB.TLS, } cfg.Checkpoint.DSN = param.ToDSN() - cfg.TiDB.Vars = make(map[string]string) - if l.cfg.To.Session != nil { - for k, v := range l.cfg.To.Session { - cfg.TiDB.Vars[k] = v - } - } - cfg.TiDB.Vars["time_zone"] = "+00:00" - cfg.TiDB.StrSQLMode = l.cfg.LoaderConfig.SQLMode if err = cfg.Adjust(ctx); err != nil { return err } - err = l.runLightning(ctx, cfg) + l.Lock() + taskCtx, cancel := context.WithCancel(ctx) + l.cancel = cancel + l.Unlock() + err = l.core.RunOnce(taskCtx, cfg, nil) if err == nil { l.finish.Store(true) - lightning.CheckpointRemove(ctx, cfg, "all") offsetSQL := l.checkPoint.GenSQL(lightningCheckpointFile, 1) err = l.toDBConns[0].executeSQL(tctx, []string{offsetSQL}) _ = l.checkPoint.UpdateOffset(lightningCheckpointFile, 1) diff --git a/syncer/util.go b/syncer/util.go index 93f9ce78ca..563314ba56 100644 --- a/syncer/util.go +++ b/syncer/util.go @@ -16,10 +16,10 @@ package syncer import ( "fmt" + "github.com/pingcap/tidb-tools/pkg/filter" dcontext "github.com/pingcap/tidb/dumpling/context" "github.com/pingcap/tidb/dumpling/export" dlog "github.com/pingcap/tidb/dumpling/log" - "github.com/pingcap/tidb-tools/pkg/filter" "github.com/pingcap/tidb/parser/ast" "go.uber.org/zap"