From 4bb34079d58d172e7b53af9e72a7d2f85f3a7f2c Mon Sep 17 00:00:00 2001 From: Steven van de Graaf Date: Fri, 23 Sep 2022 09:25:20 +0200 Subject: [PATCH 1/3] Added table name pattern option --- README.md | 6 ++++++ db_to_sqlite/cli.py | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e04ca33..9e1fc70 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ For PostgreSQL, use this: --version Show the version and exit. --all Detect and copy all tables --table TEXT Specific tables to copy + --table-name-pattern TEXT Table name pattern for tables to copy --skip TEXT When using --all skip these tables --redact TEXT... (table, column) pairs to redact with *** --sql TEXT Optional SQL query to run @@ -73,6 +74,11 @@ When running `--all` you can specify tables to skip using `--skip`: --all \ --skip=django_migrations +Alternatively, you can specify a regex pattern via `--table-name-pattern` to match against table names. For example: + + db-to-sqlite "postgresql://localhost/myblog" blog.db \ + --table-name-pattern "tag_\w+" + If you want to save the results of a custom SQL query, do this: db-to-sqlite "postgresql://localhost/myblog" output.db \ diff --git a/db_to_sqlite/cli.py b/db_to_sqlite/cli.py index cdb3037..290d778 100644 --- a/db_to_sqlite/cli.py +++ b/db_to_sqlite/cli.py @@ -1,4 +1,5 @@ import itertools +import re import click from sqlalchemy import create_engine, inspect @@ -11,6 +12,7 @@ @click.argument("path", type=click.Path(exists=False), required=True) @click.option("--all", help="Detect and copy all tables", is_flag=True) @click.option("--table", help="Specific tables to copy", multiple=True) +@click.option("--table-name-pattern", help="Table name pattern for tables to copy") @click.option("--skip", help="When using --all skip these tables", multiple=True) @click.option( "--redact", @@ -34,6 +36,7 @@ def cli( path, all, table, + table_name_pattern, skip, redact, sql, @@ -58,8 +61,10 @@ def cli( More: https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls """ - if not all and not table and not sql: - raise click.ClickException("--all OR --table OR --sql required") + if not all and not table and not table_name_pattern and not sql: + raise click.ClickException( + "--all OR --table OR --table-name-pattern OR --sql required" + ) if skip and not all: raise click.ClickException("--skip can only be used with --all") redact_columns = {} @@ -78,6 +83,12 @@ def cli( tables = table if all: tables = inspector.get_table_names() + elif table_name_pattern: + tables = [ + t + for t in inspector.get_table_names() + if re.match(table_name_pattern, t) is not None + ] if tables: foreign_keys_to_add = [] for i, table in enumerate(tables): From aa2aa023247eb3585d2f743f181df13324255a54 Mon Sep 17 00:00:00 2001 From: Steven van de Graaf Date: Fri, 23 Sep 2022 13:16:43 +0200 Subject: [PATCH 2/3] Allow `--skip` to be used with `--table-name-pattern` --- db_to_sqlite/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/db_to_sqlite/cli.py b/db_to_sqlite/cli.py index 290d778..eec1b8a 100644 --- a/db_to_sqlite/cli.py +++ b/db_to_sqlite/cli.py @@ -65,8 +65,10 @@ def cli( raise click.ClickException( "--all OR --table OR --table-name-pattern OR --sql required" ) - if skip and not all: - raise click.ClickException("--skip can only be used with --all") + if skip and not (all or table_name_pattern): + raise click.ClickException( + "--skip can only be used with --all OR --table-name-pattern" + ) redact_columns = {} for table_name, column_name in redact: redact_columns.setdefault(table_name, set()).add(column_name) From e345437abf8070f9b317c9773dd799fd749f301a Mon Sep 17 00:00:00 2001 From: Steven van de Graaf Date: Fri, 23 Sep 2022 13:20:08 +0200 Subject: [PATCH 3/3] Update README --- README.md | 12 ++++++------ db_to_sqlite/cli.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9e1fc70..102a7bb 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ For PostgreSQL, use this: --all Detect and copy all tables --table TEXT Specific tables to copy --table-name-pattern TEXT Table name pattern for tables to copy - --skip TEXT When using --all skip these tables + --skip TEXT When using --all or --table-name-pattern skip these tables --redact TEXT... (table, column) pairs to redact with *** --sql TEXT Optional SQL query to run --output TEXT Table in which to save --sql query results @@ -68,16 +68,16 @@ You can also save the data from all of your tables, effectively creating a SQLit db-to-sqlite "postgresql://localhost/myblog" blog.db \ --all -When running `--all` you can specify tables to skip using `--skip`: +Alternatively, you can specify a regex pattern via `--table-name-pattern` to match against table names. For example: db-to-sqlite "postgresql://localhost/myblog" blog.db \ - --all \ - --skip=django_migrations + --table-name-pattern "tag_\w+" -Alternatively, you can specify a regex pattern via `--table-name-pattern` to match against table names. For example: +When running `--all` or `--table-name-pattern` you can specify tables to skip using `--skip`: db-to-sqlite "postgresql://localhost/myblog" blog.db \ - --table-name-pattern "tag_\w+" + --all \ + --skip=django_migrations If you want to save the results of a custom SQL query, do this: diff --git a/db_to_sqlite/cli.py b/db_to_sqlite/cli.py index eec1b8a..5bf2b03 100644 --- a/db_to_sqlite/cli.py +++ b/db_to_sqlite/cli.py @@ -13,7 +13,7 @@ @click.option("--all", help="Detect and copy all tables", is_flag=True) @click.option("--table", help="Specific tables to copy", multiple=True) @click.option("--table-name-pattern", help="Table name pattern for tables to copy") -@click.option("--skip", help="When using --all skip these tables", multiple=True) +@click.option("--skip", help="When using --all or --table-name-pattern skip these tables", multiple=True) @click.option( "--redact", help="(table, column) pairs to redact with ***",