From 6bfcfd107d3f89b8be4c00569532d9c0dcdde611 Mon Sep 17 00:00:00 2001 From: Ermolaev Andrey Date: Wed, 3 May 2017 01:02:27 +0300 Subject: [PATCH 1/4] PRIMARY KEY only for table, FOREIGN TABLE not support it --- lib/activerecord-clean-db-structure/clean_dump.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/activerecord-clean-db-structure/clean_dump.rb b/lib/activerecord-clean-db-structure/clean_dump.rb index fc35a76..e702e6d 100644 --- a/lib/activerecord-clean-db-structure/clean_dump.rb +++ b/lib/activerecord-clean-db-structure/clean_dump.rb @@ -30,8 +30,8 @@ def run # Reduce noise for id fields by making them SERIAL instead of integer+sequence stuff # # This is a bit optimistic, but works as long as you don't have an id field thats not a sequence/uuid - dump.gsub!(/^ id integer NOT NULL,$/, ' id SERIAL PRIMARY KEY,') - dump.gsub!(/^ id bigint NOT NULL,$/, ' id BIGSERIAL PRIMARY KEY,') + dump.gsub!(/^CREATE TABLE (\w+) \(\n id integer NOT NULL,$/, "CREATE TABLE \\1 (\n id SERIAL PRIMARY KEY,") + dump.gsub!(/^CREATE TABLE (\w+) \(\n id bigint NOT NULL,$/, "CREATE TABLE \\1 (\n id BIGSERIAL PRIMARY KEY,") dump.gsub!(/^ id uuid DEFAULT uuid_generate_v4\(\) NOT NULL,$/, ' id uuid DEFAULT uuid_generate_v4() PRIMARY KEY,') dump.gsub!(/^CREATE SEQUENCE \w+_id_seq\s+START WITH 1\s+INCREMENT BY 1\s+NO MINVALUE\s+NO MAXVALUE\s+CACHE 1;$/, '') dump.gsub!(/^ALTER SEQUENCE \w+_id_seq OWNED BY .*;$/, '') From 8500408f71a7b293063c8e4ac888f5593b72fd32 Mon Sep 17 00:00:00 2001 From: Ermolaev Andrey Date: Thu, 11 May 2017 23:21:51 +0300 Subject: [PATCH 2/4] work if the id column was not listed first in the table --- .../clean_dump.rb | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/activerecord-clean-db-structure/clean_dump.rb b/lib/activerecord-clean-db-structure/clean_dump.rb index e702e6d..3a17d8e 100644 --- a/lib/activerecord-clean-db-structure/clean_dump.rb +++ b/lib/activerecord-clean-db-structure/clean_dump.rb @@ -30,8 +30,26 @@ def run # Reduce noise for id fields by making them SERIAL instead of integer+sequence stuff # # This is a bit optimistic, but works as long as you don't have an id field thats not a sequence/uuid - dump.gsub!(/^CREATE TABLE (\w+) \(\n id integer NOT NULL,$/, "CREATE TABLE \\1 (\n id SERIAL PRIMARY KEY,") - dump.gsub!(/^CREATE TABLE (\w+) \(\n id bigint NOT NULL,$/, "CREATE TABLE \\1 (\n id BIGSERIAL PRIMARY KEY,") + is_table = false, count_open_brackets = 0, count_close_brackets = 0 + @dump = dump.lines.map do |line| + is_table = true if line =~ /CREATE TABLE (\w+) \(/ + + count_open_brackets += line.count('(') + count_close_brackets += line.count(')') + + is_table = false if is_table && count_open_brackets == count_close_brackets + + if !is_table # optimization speed + line + elsif line =~ /^ id integer NOT NULL/ + line.sub('id integer NOT NULL', 'id SERIAL PRIMARY KEY') + elsif line =~ /^ id bigint NOT NULL/ + line.sub('id bigint NOT NULL', 'id BIGSERIAL PRIMARY KEY') + else + line + end + end.join + dump.gsub!(/^ id uuid DEFAULT uuid_generate_v4\(\) NOT NULL,$/, ' id uuid DEFAULT uuid_generate_v4() PRIMARY KEY,') dump.gsub!(/^CREATE SEQUENCE \w+_id_seq\s+START WITH 1\s+INCREMENT BY 1\s+NO MINVALUE\s+NO MAXVALUE\s+CACHE 1;$/, '') dump.gsub!(/^ALTER SEQUENCE \w+_id_seq OWNED BY .*;$/, '') From eddd996d026a846dbbf2807aa886a15b74c39f74 Mon Sep 17 00:00:00 2001 From: Oleg Date: Fri, 7 Jul 2017 18:13:22 +0300 Subject: [PATCH 3/4] mask user mapping (#1) --- .gitignore | 1 + Gemfile.lock | 30 +++++++++---------- .../clean_dump.rb | 10 +++++++ .../version.rb | 2 +- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 5fff1d9..b9250b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ pkg +/.idea diff --git a/Gemfile.lock b/Gemfile.lock index aaf1f25..a7a0bc5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,29 +1,29 @@ PATH remote: . specs: - activerecord-clean-db-structure (0.2.0) + activerecord-clean-db-structure (0.2.1) activerecord (>= 4.2) GEM remote: https://rubygems.org/ specs: - activemodel (5.0.1) - activesupport (= 5.0.1) - activerecord (5.0.1) - activemodel (= 5.0.1) - activesupport (= 5.0.1) - arel (~> 7.0) - activesupport (5.0.1) + activemodel (5.1.2) + activesupport (= 5.1.2) + activerecord (5.1.2) + activemodel (= 5.1.2) + activesupport (= 5.1.2) + arel (~> 8.0) + activesupport (5.1.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (~> 0.7) minitest (~> 5.1) tzinfo (~> 1.1) - arel (7.1.4) - concurrent-ruby (1.0.4) - i18n (0.8.0) - minitest (5.10.1) - thread_safe (0.3.5) - tzinfo (1.2.2) + arel (8.0.0) + concurrent-ruby (1.0.5) + i18n (0.8.4) + minitest (5.10.2) + thread_safe (0.3.6) + tzinfo (1.2.3) thread_safe (~> 0.1) PLATFORMS @@ -33,4 +33,4 @@ DEPENDENCIES activerecord-clean-db-structure! BUNDLED WITH - 1.12.5 + 1.15.1 diff --git a/lib/activerecord-clean-db-structure/clean_dump.rb b/lib/activerecord-clean-db-structure/clean_dump.rb index 3a17d8e..bdb1be8 100644 --- a/lib/activerecord-clean-db-structure/clean_dump.rb +++ b/lib/activerecord-clean-db-structure/clean_dump.rb @@ -27,6 +27,16 @@ def run # Remove useless comment lines dump.gsub!(/^--$/, '') + # Mask user mapping + dump.gsub!( + /^CREATE USER MAPPING FOR \w+ SERVER (\w+) .*;/m, + 'CREATE USER MAPPING FOR some_user SERVER \1;' + ) + dump.gsub!( + /^-- Name: USER MAPPING \w+ SERVER (\w+); Type: USER MAPPING/, + '-- Name: USER MAPPING some_user SERVER \1; Type: USER MAPPING' + ) + # Reduce noise for id fields by making them SERIAL instead of integer+sequence stuff # # This is a bit optimistic, but works as long as you don't have an id field thats not a sequence/uuid diff --git a/lib/activerecord-clean-db-structure/version.rb b/lib/activerecord-clean-db-structure/version.rb index 580d9e3..1b7f390 100644 --- a/lib/activerecord-clean-db-structure/version.rb +++ b/lib/activerecord-clean-db-structure/version.rb @@ -1,3 +1,3 @@ module ActiveRecordCleanDbStructure - VERSION = '0.2.0' + VERSION = '0.2.1' end From c1c5f0451d59aad3412f50dcc0bd9f3074689e55 Mon Sep 17 00:00:00 2001 From: Ermolaev Andrey Date: Fri, 7 Jul 2017 19:13:18 +0300 Subject: [PATCH 4/4] refactoring and optimization --- .../clean_dump.rb | 31 +++---------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/lib/activerecord-clean-db-structure/clean_dump.rb b/lib/activerecord-clean-db-structure/clean_dump.rb index bdb1be8..44059b3 100644 --- a/lib/activerecord-clean-db-structure/clean_dump.rb +++ b/lib/activerecord-clean-db-structure/clean_dump.rb @@ -28,37 +28,14 @@ def run dump.gsub!(/^--$/, '') # Mask user mapping - dump.gsub!( - /^CREATE USER MAPPING FOR \w+ SERVER (\w+) .*;/m, - 'CREATE USER MAPPING FOR some_user SERVER \1;' - ) - dump.gsub!( - /^-- Name: USER MAPPING \w+ SERVER (\w+); Type: USER MAPPING/, - '-- Name: USER MAPPING some_user SERVER \1; Type: USER MAPPING' - ) + dump.gsub!(/^CREATE USER MAPPING FOR \w+ SERVER (\w+) .*?;/m, 'CREATE USER MAPPING FOR some_user SERVER \1;') + dump.gsub!(/^-- Name: USER MAPPING \w+ SERVER (\w+); Type: USER MAPPING/, '-- Name: USER MAPPING some_user SERVER \1; Type: USER MAPPING') # Reduce noise for id fields by making them SERIAL instead of integer+sequence stuff # # This is a bit optimistic, but works as long as you don't have an id field thats not a sequence/uuid - is_table = false, count_open_brackets = 0, count_close_brackets = 0 - @dump = dump.lines.map do |line| - is_table = true if line =~ /CREATE TABLE (\w+) \(/ - - count_open_brackets += line.count('(') - count_close_brackets += line.count(')') - - is_table = false if is_table && count_open_brackets == count_close_brackets - - if !is_table # optimization speed - line - elsif line =~ /^ id integer NOT NULL/ - line.sub('id integer NOT NULL', 'id SERIAL PRIMARY KEY') - elsif line =~ /^ id bigint NOT NULL/ - line.sub('id bigint NOT NULL', 'id BIGSERIAL PRIMARY KEY') - else - line - end - end.join + dump.gsub!(/^CREATE TABLE (\w+) \(\n id integer NOT NULL(.*?);/m, "CREATE TABLE \\1 (\n id SERIAL PRIMARY KEY\\2;") + dump.gsub!(/^CREATE TABLE (\w+) \(\n id bigint NOT NULL(.*?);/m, "CREATE TABLE \\1 (\n id BIGSERIAL PRIMARY KEY\\2;") dump.gsub!(/^ id uuid DEFAULT uuid_generate_v4\(\) NOT NULL,$/, ' id uuid DEFAULT uuid_generate_v4() PRIMARY KEY,') dump.gsub!(/^CREATE SEQUENCE \w+_id_seq\s+START WITH 1\s+INCREMENT BY 1\s+NO MINVALUE\s+NO MAXVALUE\s+CACHE 1;$/, '')