diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..64ca431 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,4 @@ +[report] +exclude_lines = + # Don't complain if tests don't hit defensive assertion code: + raise NotImplementedError \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index e0ff611..1dbe8ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,15 @@ ## Development +## 0.6.0 (2021-07-13) + +* [#28](https://github.com/rheinwerk-verlag/postgresql-anonymizer/pull/25): Add json support ([nurikk](https://github.com/nurikk)) +* [#27](https://github.com/rheinwerk-verlag/postgresql-anonymizer/pull/25): Better anonymisation ([nurikk](https://github.com/nurikk)) +* [#25](https://github.com/rheinwerk-verlag/postgresql-anonymizer/pull/25): Remove column specification for `cursor.copy_from` call ([nurikk](https://github.com/nurikk)) + ## 0.5.0 (2021-06-30) -* [#22](https://github.com/rheinwerk-verlag/postgresql-anonymizer/pull/22): Fix table and column name quotes in cursor.copy_from call ([nurikk](https://github.com/nurikk)) +* [#22](https://github.com/rheinwerk-verlag/postgresql-anonymizer/pull/22): Fix table and column name quotes in `cursor.copy_from` call ([nurikk](https://github.com/nurikk)) * [#23](https://github.com/rheinwerk-verlag/postgresql-anonymizer/pull/23): Allow uniq faker ([nurikk](https://github.com/nurikk)) ## 0.4.1 (2021-05-27) diff --git a/README.rst b/README.rst index df992d5..f1e4884 100644 --- a/README.rst +++ b/README.rst @@ -21,25 +21,30 @@ Features * Exclude data for anonymization depending on regular expressions * Truncate entire tables for unwanted data -+----------------+----------------------+-----------------------+----------------------------------+ -| Field | Value | Provider | Output | -+================+======================+=======================+==================================+ -| ``first_name`` | John | ``choice`` | (Bob|Larry|Lisa) | -+----------------+----------------------+-----------------------+----------------------------------+ -| ``title`` | Dr. | ``clear`` | | -+----------------+----------------------+-----------------------+----------------------------------+ -| ``street`` | Irving St | ``faker.street_name`` | Miller Station | -+----------------+----------------------+-----------------------+----------------------------------+ -| ``password`` | dsf82hFxcM | ``mask`` | XXXXXXXXXX | -+----------------+----------------------+-----------------------+----------------------------------+ -| ``email`` | jane.doe@example.com | ``md5`` | 0cba00ca3da1b283a57287bcceb17e35 | -+----------------+----------------------+-----------------------+----------------------------------+ -| ``email`` | jane.doe@example.com | ``faker.unique.email``| alex7@sample.com | -+----------------+----------------------+-----------------------+----------------------------------+ -| ``ip`` | 157.50.1.20 | ``set`` | 127.0.0.1 | -+----------------+----------------------+-----------------------+----------------------------------+ ++----------------+----------------------+------------------------+----------------------------------+ +| Field | Value | Provider | Output | ++================+======================+========================+==================================+ +| ``first_name`` | John | ``choice`` | (Bob|Larry|Lisa) | ++----------------+----------------------+------------------------+----------------------------------+ +| ``title`` | Dr. | ``clear`` | | ++----------------+----------------------+------------------------+----------------------------------+ +| ``street`` | Irving St | ``faker.street_name`` | Miller Station | ++----------------+----------------------+------------------------+----------------------------------+ +| ``password`` | dsf82hFxcM | ``mask`` | XXXXXXXXXX | ++----------------+----------------------+------------------------+----------------------------------+ +| ``email`` | jane.doe@example.com | ``md5`` | 0cba00ca3da1b283a57287bcceb17e35 | ++----------------+----------------------+------------------------+----------------------------------+ +| ``email`` | jane.doe@example.com | ``faker.unique.email`` | alex7@sample.com | ++----------------+----------------------+------------------------+----------------------------------+ +| ``phone_num`` | 65923473 | ``md5``as_number: True | 3948293448 | ++----------------+----------------------+------------------------+----------------------------------+ +| ``ip`` | 157.50.1.20 | ``set`` | 127.0.0.1 | ++----------------+----------------------+------------------------+----------------------------------+ +| ``uuid_col`` | 00010203-0405-...... | ``uuid4`` | f7c1bd87-4d.... | ++----------------+----------------------+------------------------+----------------------------------+ Note: `faker.unique.[provider]` only supported on python3.5+ (Faker library min supported python version) +Note: `uuid4` - only for (native `uuid4`) columns See the `documentation`_ for a more detailed description of the provided anonymization methods. @@ -75,11 +80,13 @@ Usage --dry-run Don't commit changes made on the database --dump-file DUMP_FILE Create a database dump file with the given name + --init-sql INIT_SQL SQL to run before starting anonymization Despite the database connection values, you will have to define a YAML schema file, that includes all anonymization rules for that database. Take a look at the `schema documentation`_ or the `YAML sample schema`_. + Example call:: $ pganonymize --schema=myschema.yml \ @@ -89,6 +96,14 @@ Example call:: --host=db.host.example.com \ -v + $ pganonymize --schema=myschema.yml \ + --dbname=test_database \ + --user=username \ + --password=mysecret \ + --host=db.host.example.com \ + --init-sql "set search_path to non_public_search_path; set work_mem to '1GB';" \ + -v + Database dump ~~~~~~~~~~~~~ diff --git a/pganonymizer/__main__.py b/pganonymizer/__main__.py index 9e40e33..a18a742 100644 --- a/pganonymizer/__main__.py +++ b/pganonymizer/__main__.py @@ -5,9 +5,11 @@ def main(): - from pganonymizer.cli import main + from pganonymizer.cli import get_arg_parser, main + try: - main() + args = get_arg_parser().parse_args() + main(args) exit_status = 0 except KeyboardInterrupt: exit_status = 1 diff --git a/pganonymizer/cli.py b/pganonymizer/cli.py index 2b1ce65..4f6eab7 100644 --- a/pganonymizer/cli.py +++ b/pganonymizer/cli.py @@ -4,7 +4,6 @@ import argparse import logging -import sys import time import yaml @@ -33,8 +32,7 @@ def list_provider_classes(): print('{:<10} {}'.format(provider_cls.id, provider_cls.__doc__)) -def main(): - """Main method""" +def get_arg_parser(): parser = argparse.ArgumentParser(description='Anonymize data of a PostgreSQL database') parser.add_argument('-v', '--verbose', action='count', help='Increase verbosity') parser.add_argument('-l', '--list-providers', action='store_true', help='Show a list of all available providers', @@ -49,8 +47,13 @@ def main(): parser.add_argument('--dry-run', action='store_true', help='Don\'t commit changes made on the database', default=False) parser.add_argument('--dump-file', help='Create a database dump file with the given name') + parser.add_argument('--init-sql', help='SQL to run before starting anonymization', default=False) + + return parser + - args = parser.parse_args() +def main(args): + """Main method""" loglevel = logging.WARNING if args.verbose: @@ -59,16 +62,21 @@ def main(): if args.list_providers: list_provider_classes() - sys.exit(0) + return 0 schema = yaml.load(open(args.schema), Loader=yaml.FullLoader) pg_args = get_pg_args(args) connection = get_connection(pg_args) + if args.init_sql: + cursor = connection.cursor() + logging.info('Executing initialisation sql {}'.format(args.init_sql)) + cursor.execute(args.init_sql) + cursor.close() start_time = time.time() truncate_tables(connection, schema.get('truncate', [])) - anonymize_tables(connection, schema.get('tables', []), verbose=args.verbose) + anonymize_tables(connection, schema.get('tables', []), verbose=args.verbose, dry_run=args.dry_run) if not args.dry_run: connection.commit() @@ -79,7 +87,3 @@ def main(): if args.dump_file: create_database_dump(args.dump_file, pg_args) - - -if __name__ == '__main__': - main() diff --git a/pganonymizer/constants.py b/pganonymizer/constants.py index 0eb14d8..175782d 100644 --- a/pganonymizer/constants.py +++ b/pganonymizer/constants.py @@ -4,11 +4,8 @@ # Default name for the primary key column DEFAULT_PRIMARY_KEY = 'id' -# Delimiter used to buffer and import database data. -COPY_DB_DELIMITER = '\x1f' - # Filename of the default schema DEFAULT_SCHEMA_FILE = 'schema.yml' # Default chunk size for data fetch -DEFAULT_CHUNK_SIZE = 2000 +DEFAULT_CHUNK_SIZE = 100000 diff --git a/pganonymizer/providers.py b/pganonymizer/providers.py index 79c09e6..f34006e 100644 --- a/pganonymizer/providers.py +++ b/pganonymizer/providers.py @@ -1,6 +1,7 @@ import operator import random from hashlib import md5 +from uuid import uuid4 from faker import Faker from six import with_metaclass @@ -111,9 +112,16 @@ class MD5Provider(with_metaclass(ProviderMeta, Provider)): """Provider to hash a value with the md5 algorithm.""" id = 'md5' + default_max_length = 8 def alter_value(self, value): - return md5(value.encode('utf-8')).hexdigest() + as_number = self.kwargs.get('as_number', False) + as_number_length = self.kwargs.get('as_number_length', self.default_max_length) + hashed = md5(value.encode('utf-8')).hexdigest() + if as_number: + return int(hashed, 16) % (10 ** as_number_length) + else: + return hashed class SetProvider(with_metaclass(ProviderMeta, Provider)): @@ -123,3 +131,12 @@ class SetProvider(with_metaclass(ProviderMeta, Provider)): def alter_value(self, value): return self.kwargs.get('value') + + +class UUID4Provider(with_metaclass(ProviderMeta, Provider)): + """Provider to set a random uuid value.""" + + id = 'uuid4' + + def alter_value(self, value): + return uuid4() diff --git a/pganonymizer/utils.py b/pganonymizer/utils.py index 7f9866d..faa5aa6 100644 --- a/pganonymizer/utils.py +++ b/pganonymizer/utils.py @@ -2,93 +2,125 @@ from __future__ import absolute_import -import csv import json import logging +import math import re import subprocess +import time +import parmap +from pgcopy import CopyManager import psycopg2 import psycopg2.extras -from progress.bar import IncrementalBar -from psycopg2.errors import BadCopyFileFormat, InvalidTextRepresentation -from six import StringIO +from psycopg2.sql import SQL, Identifier, Composed +from tqdm import trange -from pganonymizer.constants import COPY_DB_DELIMITER, DEFAULT_CHUNK_SIZE, DEFAULT_PRIMARY_KEY -from pganonymizer.exceptions import BadDataFormat +from pganonymizer.constants import DEFAULT_CHUNK_SIZE, DEFAULT_PRIMARY_KEY from pganonymizer.providers import get_provider -def anonymize_tables(connection, definitions, verbose=False): +def anonymize_tables(connection, definitions, verbose=False, dry_run=False): """ Anonymize a list of tables according to the schema definition. :param connection: A database connection instance. :param list definitions: A list of table definitions from the YAML schema. :param bool verbose: Display logging information and a progress bar. + :param bool dry_run: Script is runnin in dry-run mode, no commit expected. """ for definition in definitions: + start_time = time.time() table_name = list(definition.keys())[0] logging.info('Found table definition "%s"', table_name) table_definition = definition[table_name] columns = table_definition.get('fields', []) excludes = table_definition.get('excludes', []) search = table_definition.get('search') - column_dict = get_column_dict(columns) primary_key = table_definition.get('primary_key', DEFAULT_PRIMARY_KEY) - total_count = get_table_count(connection, table_name) + total_count = get_table_count(connection, table_name, dry_run) chunk_size = table_definition.get('chunk_size', DEFAULT_CHUNK_SIZE) - data, table_columns = build_data(connection, table_name, columns, excludes, search, total_count, chunk_size, - verbose) - import_data(connection, column_dict, table_name, table_columns, primary_key, data) + build_and_then_import_data(connection, table_name, primary_key, columns, excludes, + search, total_count, chunk_size, verbose=verbose, dry_run=dry_run) + end_time = time.time() + logging.info('{} anonymization took {:.2f}s'.format(table_name, end_time - start_time)) -def build_data(connection, table, columns, excludes, search, total_count, chunk_size, verbose=False): +def process_row(row, columns, excludes): + if row_matches_excludes(row, excludes): + return None + else: + row_column_dict = get_column_values(row, columns) + if row_column_dict: + for key, value in row_column_dict.items(): + row[key] = value + else: + return None + return row + + +def build_and_then_import_data(connection, table, primary_key, columns, + excludes, search, total_count, chunk_size, verbose=False, dry_run=False): """ Select all data from a table and return it together with a list of table columns. :param connection: A database connection instance. :param str table: Name of the table to retrieve the data. + :param str primary_key: Table primary key :param list columns: A list of table fields :param list[dict] excludes: A list of exclude definitions. :param str search: A SQL WHERE (search_condition) to filter and keep only the searched rows. :param int total_count: The amount of rows for the current table :param int chunk_size: Number of data rows to fetch with the cursor :param bool verbose: Display logging information and a progress bar. - :return: A tuple containing the data list and a complete list of all table columns. - :rtype: (list, list) + :param bool dry_run: Script is runnin in dry-run mode, no commit expected. """ - if verbose: - progress_bar = IncrementalBar('Anonymizing', max=total_count) - sql_select = "SELECT * FROM {table}".format(table=table) + column_names = get_column_names(columns) + sql_columns = SQL(', ').join([Identifier(column_name) for column_name in [primary_key] + column_names]) + sql_select = SQL('SELECT {columns} FROM {table}').format(table=Identifier(table), columns=sql_columns) if search: - sql = "{select} WHERE {search_condition};".format(select=sql_select, search_condition=search) - else: - sql = "{select};".format(select=sql_select) + sql_select = Composed([sql_select, SQL(" WHERE {search_condition}".format(search_condition=search))]) + if dry_run: + sql_select = Composed([sql_select, SQL(" LIMIT 100")]) cursor = connection.cursor(cursor_factory=psycopg2.extras.DictCursor, name='fetch_large_result') - cursor.execute(sql) - data = [] - table_columns = None - while True: + cursor.execute(sql_select.as_string(connection)) + temp_table = 'tmp_{table}'.format(table=table) + create_temporary_table(connection, columns, table, temp_table, primary_key) + batches = int(math.ceil((1.0 * total_count) / (1.0 * chunk_size))) + for i in trange(batches, desc="Processing {} batches for {}".format(batches, table), disable=not verbose): records = cursor.fetchmany(size=chunk_size) - if not records: - break - for row in records: - row_column_dict = {} - if not row_matches_excludes(row, excludes): - row_column_dict = get_column_values(row, columns) - for key, value in row_column_dict.items(): - row[key] = value - if verbose: - progress_bar.next() - table_columns = [column for column in row.keys()] - if not row_column_dict: - continue - data.append(row.values()) - if verbose: - progress_bar.finish() + if records: + data = parmap.map(process_row, records, columns, excludes, pm_pbar=verbose) + import_data(connection, temp_table, [primary_key] + column_names, filter(None, data)) + apply_anonymized_data(connection, temp_table, table, primary_key, columns) + + cursor.close() + + +def apply_anonymized_data(connection, temp_table, source_table, primary_key, definitions): + logging.info('Applying changes on table {}'.format(source_table)) + cursor = connection.cursor() + create_index_sql = SQL('CREATE INDEX ON {temp_table} ({primary_key})') + sql = create_index_sql.format(temp_table=Identifier(temp_table), primary_key=Identifier(primary_key)) + cursor.execute(sql.as_string(connection)) + + column_names = get_column_names(definitions) + columns_identifiers = [SQL('{column} = s.{column}').format(column=Identifier(column)) for column in column_names] + set_columns = SQL(', ').join(columns_identifiers) + sql_args = { + "table": Identifier(source_table), + "columns": set_columns, + "source": Identifier(temp_table), + "primary_key": Identifier(primary_key) + } + sql = SQL( + 'UPDATE {table} t ' + 'SET {columns} ' + 'FROM {source} s ' + 'WHERE t.{primary_key} = s.{primary_key}' + ).format(**sql_args) + cursor.execute(sql.as_string(connection)) cursor.close() - return data, table_columns def row_matches_excludes(row, excludes=None): @@ -115,53 +147,30 @@ def row_matches_excludes(row, excludes=None): return False -def copy_from(connection, data, table, columns): - """ - Copy the data from a table to a temporary table. - - :param connection: A database connection instance. - :param list data: The data of a table. - :param str table: Name of the temporary table used for copying the data. - :param list columns: All columns of the current table. - :raises BadDataFormat: If the data cannot be imported due to a invalid format. - """ - new_data = data2csv(data) +def create_temporary_table(connection, definitions, source_table, temp_table, primary_key): + primary_key = primary_key if primary_key else DEFAULT_PRIMARY_KEY + column_names = get_column_names(definitions) + sql_columns = SQL(', ').join([Identifier(column_name) for column_name in [primary_key] + column_names]) + ctas_query = SQL("""CREATE TEMP TABLE {temp_table} AS SELECT {columns} + FROM {source_table} WITH NO DATA""") cursor = connection.cursor() - try: - quoted_cols = ['"{}"'.format(column) for column in columns] - cursor.copy_from(new_data, table, sep=COPY_DB_DELIMITER, null='\\N', columns=quoted_cols) - except (BadCopyFileFormat, InvalidTextRepresentation) as exc: - raise BadDataFormat(exc) + cursor.execute(ctas_query.format(temp_table=Identifier(temp_table), + source_table=Identifier(source_table), columns=sql_columns) + .as_string(connection) + ) cursor.close() -def import_data(connection, column_dict, source_table, table_columns, primary_key, data): +def import_data(connection, table_name, column_names, data): """ Import the temporary and anonymized data to a temporary table and write the changes back. - :param connection: A database connection instance. - :param dict column_dict: A dictionary with all columns (specified by the schema definition) and a default value of - None. - :param str source_table: Name of the table to be anonymized. - :param list table_columns: A list of all table columns. - :param str primary_key: Name of the tables primary key. + :param str table_name: Name of the table to be populated with data. + :param list column_names: A list of table fields :param list data: The table data. """ - primary_key = primary_key if primary_key else DEFAULT_PRIMARY_KEY - temp_table = 'tmp_{table}'.format(table=source_table) - cursor = connection.cursor() - cursor.execute('CREATE TEMP TABLE "%s" (LIKE %s INCLUDING ALL) ON COMMIT DROP;' % (temp_table, source_table)) - copy_from(connection, data, temp_table, table_columns) - set_columns = ', '.join(['{column} = s.{column}'.format(column='"{}"'.format(key)) for key in column_dict.keys()]) - sql = ( - 'UPDATE {table} t ' - 'SET {columns} ' - 'FROM {source} s ' - 'WHERE t.{primary_key} = s.{primary_key};' - ).format(table=source_table, columns=set_columns, source=temp_table, primary_key=primary_key) - cursor.execute(sql) - cursor.execute('DROP TABLE %s;' % temp_table) - cursor.close() + mgr = CopyManager(connection, table_name, column_names) + mgr.copy([[escape_str_replace(val) for col, val in row.items()] for row in data]) def get_connection(pg_args): @@ -175,70 +184,25 @@ def get_connection(pg_args): return psycopg2.connect(**pg_args) -def get_table_count(connection, table): +def get_table_count(connection, table, dry_run): """ Return the number of table entries. :param connection: A database connection instance :param str table: Name of the database table + :param bool dry_run: Script is runnin in dry-run mode, no commit expected. :return: The number of table entries :rtype: int """ - sql = 'SELECT COUNT(*) FROM {table};'.format(table=table) - cursor = connection.cursor() - cursor.execute(sql) - total_count = cursor.fetchone()[0] - cursor.close() - return total_count - - -def data2csv(data): - """ - Return a string buffer, that contains delimited data. - - :param list data: A list of values - :return: A stream that contains tab delimited csv data - :rtype: StringIO - """ - buf = StringIO() - writer = csv.writer(buf, delimiter=COPY_DB_DELIMITER, lineterminator='\n', quotechar='~') - for row in data: - row_data = [] - for x in row: - if x is None: - val = '\\N' - elif type(x) == str: - val = x.strip() - elif type(x) == dict: - val = json.dumps(x) - else: - val = x - row_data.append(val) - writer.writerow(row_data) - buf.seek(0) - return buf - - -def get_column_dict(columns): - """ - Return a dictionary with all fields from the table definition and None as value. - - :param list columns: A list of field definitions from the YAML schema, e.g.: - - >>> [ - >>> {'first_name': {'provider': 'set', 'value': 'Foo'}}, - >>> {'guest_email': {'append': '@localhost', 'provider': 'md5'}}, - >>> ] - - :return: A dictionary containing all fields to be altered with a default value of None, e.g.:: - {'guest_email': None} - :rtype: dict - """ - column_dict = {} - for definition in columns: - column_name = list(definition.keys())[0] - column_dict[column_name] = None - return column_dict + if dry_run: + return 100 + else: + sql = SQL('SELECT COUNT(*) FROM {table}').format(table=Identifier(table)) + cursor = connection.cursor() + cursor.execute(sql.as_string(connection)) + total_count = cursor.fetchone()[0] + cursor.close() + return total_count def get_column_values(row, columns): @@ -258,19 +222,23 @@ def get_column_values(row, columns): """ column_dict = {} for definition in columns: - column_name = list(definition.keys())[0] - column_definition = definition[column_name] + full_column_name = get_column_name(definition, True) + column_name = get_column_name(definition, False) + column_definition = definition[full_column_name] provider_config = column_definition.get('provider') - orig_value = row.get(column_name) - if not orig_value: - # Skip the current column if there is no value to be altered - continue - provider = get_provider(provider_config) - value = provider.alter_value(orig_value) - append = column_definition.get('append') - if append: - value = value + append - column_dict[column_name] = value + orig_value = nested_get(row, full_column_name) + # Skip the current column if there is no value to be altered + if orig_value is not None: + provider = get_provider(provider_config) + value = provider.alter_value(orig_value) + append = column_definition.get('append') + if append: + value = value + append + format = column_definition.get('format') + if format: + value = format.format(pga_value=value, **row) + nested_set(row, full_column_name, value) + column_dict[column_name] = nested_get(row, column_name) return column_dict @@ -284,9 +252,9 @@ def truncate_tables(connection, tables): if not tables: return cursor = connection.cursor() - table_names = ', '.join(tables) + table_names = SQL(', ').join([Identifier(table_name) for table_name in tables]) logging.info('Truncating tables "%s"', table_names) - cursor.execute('TRUNCATE TABLE {tables};'.format(tables=table_names)) + cursor.execute(SQL('TRUNCATE TABLE {tables}').format(tables=table_names).as_string(connection)) cursor.close() @@ -304,3 +272,81 @@ def create_database_dump(filename, db_args): ) logging.info('Creating database dump file "%s"', filename) subprocess.run(cmd, shell=True) + + +def get_column_name(definition, fully_qualified=False): + """ + Get column name by definition. + + :param dict definition: Column definition + :param bool fully_qualified: Get complete column name with path (json objects) + :return: A string, containing column name. ex: + id + name + metadata.col1 + :rtype: string + """ + col_name = list(definition.keys())[0] + if fully_qualified: + return col_name + else: + return col_name.split('.', 2)[0] + + +def get_column_names(definitions): + """Get disctinct column names from definitions + + :param list definitions: A list of table definitions from the YAML schema. + :return: A list of column names + :rtype: list + """ + names = [] + for definition in definitions: + name = get_column_name(definition) + if name not in names: + names.append(name) + return names + + +def escape_str_replace(value): + """Get escaped value + + :param Value to be encoded. + :return: Escaped value + :rtype: unknown + """ + if isinstance(value, dict): + return json.dumps(value).encode() + return value + + +def nested_get(dic, path, delimiter='.'): + """Get from dictionary by path + + :dic dict Source dictionaly. + :path string Path withing dictionary + :delimiter string Path delimiter + :return: Value at path + :rtype: unknown + """ + try: + keys = path.split(delimiter) + for key in keys[:-1]: + dic = dic.get(key, {}) + return dic[keys[-1]] + except (AttributeError, KeyError, TypeError): + return None + + +def nested_set(dic, path, value, delimiter='.'): + """Set dictionary value by path + + :dic dict Source dictionaly. + :path string Path withing dictionary + :value unknow Value to be set + :delimiter string Path delimiter + """ + keys = path.split(delimiter) + for key in keys[:-1]: + dic = dic.get(key, {}) + dic[keys[-1]] = value diff --git a/pganonymizer/version.py b/pganonymizer/version.py index 7568cb9..1ab444e 100644 --- a/pganonymizer/version.py +++ b/pganonymizer/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -__version__ = '0.5.0' +__version__ = '0.6.0' diff --git a/poetry.lock b/poetry.lock index ad5d1bb..14459d1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6,14 +6,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "apipkg" -version = "1.5" -description = "apipkg: namespace control and lazy-import mechanism" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - [[package]] name = "appnope" version = "0.1.2" @@ -32,17 +24,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "20.3.0" +version = "21.2.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] -docs = ["furo", "sphinx", "zope.interface"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] [[package]] name = "babel" @@ -85,7 +77,7 @@ python-versions = "*" [[package]] name = "certifi" -version = "2020.12.5" +version = "2021.5.30" description = "Python package for providing Mozilla's CA Bundle." category = "dev" optional = false @@ -103,7 +95,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" name = "colorama" version = "0.4.4" description = "Cross-platform colored terminal text." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -154,7 +146,15 @@ toml = ["toml"] [[package]] name = "decorator" -version = "5.0.7" +version = "4.4.2" +description = "Decorators for Humans" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*" + +[[package]] +name = "decorator" +version = "5.0.9" description = "Decorators for Humans" category = "dev" optional = false @@ -178,15 +178,12 @@ python-versions = "*" [[package]] name = "execnet" -version = "1.8.0" +version = "1.9.0" description = "execnet: rapid multi-Python deployment" category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -[package.dependencies] -apipkg = ">=1.4" - [package.extras] testing = ["pre-commit"] @@ -218,7 +215,7 @@ text-unidecode = "1.3" [[package]] name = "flake8" -version = "3.9.1" +version = "3.9.2" description = "the modular source code checker: pep8 pyflakes and co" category = "dev" optional = false @@ -488,7 +485,7 @@ six = ">=1.0.0,<2.0.0" [[package]] name = "more-itertools" -version = "8.7.0" +version = "8.8.0" description = "More routines for operating on iterables, beyond itertools" category = "dev" optional = false @@ -505,6 +502,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] pyparsing = ">=2.0.2" +[[package]] +name = "parmap" +version = "1.5.2" +description = "map and starmap implementations passing additional arguments and parallelizing if possible" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +progress_bar = ["tqdm (>=4.8.4)"] + [[package]] name = "parso" version = "0.7.1" @@ -518,7 +526,7 @@ testing = ["docopt", "pytest (>=3.0.7)"] [[package]] name = "pathlib2" -version = "2.3.5" +version = "2.3.6" description = "Object-oriented filesystem paths" category = "dev" optional = false @@ -547,6 +555,18 @@ python-versions = "*" [package.dependencies] ptyprocess = ">=0.5" +[[package]] +name = "pgcopy" +version = "1.5.0" +description = "Fast db insert with postgresql binary copy" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +psycopg2 = "*" +pytz = "*" + [[package]] name = "pickleshare" version = "0.7.5" @@ -572,14 +592,6 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] dev = ["pre-commit", "tox"] -[[package]] -name = "progress" -version = "1.5" -description = "Easy to use progress bars" -category = "main" -optional = false -python-versions = "*" - [[package]] name = "prompt-toolkit" version = "1.0.18" @@ -604,6 +616,14 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" six = ">=1.9.0" wcwidth = "*" +[[package]] +name = "psycopg2" +version = "2.8.6" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +category = "main" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" + [[package]] name = "psycopg2-binary" version = "2.8.6" @@ -702,7 +722,7 @@ pytest = ">=2.2" [[package]] name = "pytest-cov" -version = "2.11.1" +version = "2.12.1" description = "Pytest plugin for measuring coverage." category = "dev" optional = false @@ -711,9 +731,10 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] coverage = ">=5.2.1" pytest = ">=4.6" +toml = "*" [package.extras] -testing = ["fields", "hunter", "process-tests (==2.0.2)", "six", "pytest-xdist", "virtualenv"] +testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"] [[package]] name = "pytest-pep8" @@ -754,7 +775,7 @@ six = ">=1.5" name = "pytz" version = "2021.1" description = "World timezone definitions, modern and historical" -category = "dev" +category = "main" optional = false python-versions = "*" @@ -815,7 +836,7 @@ python-versions = "*" [[package]] name = "six" -version = "1.15.0" +version = "1.16.0" description = "Python 2 and 3 compatibility utilities" category = "main" optional = false @@ -886,6 +907,30 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "tqdm" +version = "4.61.2" +description = "Fast, Extensible Progress Meter" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +telegram = ["requests"] + [[package]] name = "traitlets" version = "4.3.3" @@ -913,16 +958,16 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <3.5" [[package]] name = "urllib3" -version = "1.26.4" +version = "1.26.6" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] +brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] -brotli = ["brotlipy (>=0.6.0)"] [[package]] name = "wcwidth" @@ -961,17 +1006,13 @@ testing = ["pathlib2", "unittest2", "jaraco.itertools", "func-timeout"] [metadata] lock-version = "1.1" python-versions = "~2.7 || ^3.5" -content-hash = "dfe5d3577a1b26314b5e08b1407ee29a8d43824f6f17fcec9d60ab280566af1f" +content-hash = "b750ea022c55d15d6bc186be5ee1778b2ea4cb5d4bf910686d6685c4ca5c0950" [metadata.files] alabaster = [ {file = "alabaster-0.7.12-py2.py3-none-any.whl", hash = "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359"}, {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"}, ] -apipkg = [ - {file = "apipkg-1.5-py2.py3-none-any.whl", hash = "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c"}, - {file = "apipkg-1.5.tar.gz", hash = "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6"}, -] appnope = [ {file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"}, {file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"}, @@ -981,8 +1022,8 @@ atomicwrites = [ {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ - {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, - {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, + {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, + {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, ] babel = [ {file = "Babel-2.9.1-py2.py3-none-any.whl", hash = "sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9"}, @@ -1001,8 +1042,8 @@ backcall = [ {file = "backports.shutil_get_terminal_size-1.0.0.tar.gz", hash = "sha256:713e7a8228ae80341c70586d1cc0a8caa5207346927e23d09dcbcaf18eadec80"}, ] certifi = [ - {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"}, - {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"}, + {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, + {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, ] chardet = [ {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, @@ -1079,8 +1120,10 @@ coverage = [ {file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"}, ] decorator = [ - {file = "decorator-5.0.7-py3-none-any.whl", hash = "sha256:945d84890bb20cc4a2f4a31fc4311c0c473af65ea318617f13a7257c9a58bc98"}, - {file = "decorator-5.0.7.tar.gz", hash = "sha256:6f201a6c4dac3d187352661f508b9364ec8091217442c9478f1f83c003a0f060"}, + {file = "decorator-4.4.2-py2.py3-none-any.whl", hash = "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760"}, + {file = "decorator-4.4.2.tar.gz", hash = "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7"}, + {file = "decorator-5.0.9-py3-none-any.whl", hash = "sha256:6e5c199c16f7a9f0e3a61a4a54b3d27e7dad0dbdde92b944426cb20914376323"}, + {file = "decorator-5.0.9.tar.gz", hash = "sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5"}, ] docutils = [ {file = "docutils-0.17.1-py2.py3-none-any.whl", hash = "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61"}, @@ -1092,8 +1135,8 @@ enum34 = [ {file = "enum34-1.1.10.tar.gz", hash = "sha256:cce6a7477ed816bd2542d03d53db9f0db935dd013b70f336a95c73979289f248"}, ] execnet = [ - {file = "execnet-1.8.0-py2.py3-none-any.whl", hash = "sha256:7a13113028b1e1cc4c6492b28098b3c6576c9dccc7973bfe47b342afadafb2ac"}, - {file = "execnet-1.8.0.tar.gz", hash = "sha256:b73c5565e517f24b62dea8a5ceac178c661c4309d3aa0c3e420856c072c411b4"}, + {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, + {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, ] faker = [ {file = "Faker-3.0.1-py2.py3-none-any.whl", hash = "sha256:6eb3581e990e36ef6f1cf37f70f9a799e119e1a7b94a6062a14f1b8d781c67e4"}, @@ -1102,8 +1145,8 @@ faker = [ {file = "Faker-4.18.0.tar.gz", hash = "sha256:6279746aed175a693108238e6d1ab8d7e26d0ec7ff8474f61025b9fdaae15d65"}, ] flake8 = [ - {file = "flake8-3.9.1-py2.py3-none-any.whl", hash = "sha256:3b9f848952dddccf635be78098ca75010f073bfe14d2c6bda867154bea728d2a"}, - {file = "flake8-3.9.1.tar.gz", hash = "sha256:1aa8990be1e689d96c745c5682b687ea49f2e05a443aff1f8251092b0014e378"}, + {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, + {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, ] funcsigs = [ {file = "funcsigs-1.0.2-py2.py3-none-any.whl", hash = "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca"}, @@ -1228,20 +1271,24 @@ more-itertools = [ {file = "more-itertools-5.0.0.tar.gz", hash = "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4"}, {file = "more_itertools-5.0.0-py2-none-any.whl", hash = "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc"}, {file = "more_itertools-5.0.0-py3-none-any.whl", hash = "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9"}, - {file = "more-itertools-8.7.0.tar.gz", hash = "sha256:c5d6da9ca3ff65220c3bfd2a8db06d698f05d4d2b9be57e1deb2be5a45019713"}, - {file = "more_itertools-8.7.0-py3-none-any.whl", hash = "sha256:5652a9ac72209ed7df8d9c15daf4e1aa0e3d2ccd3c87f8265a0673cd9cbc9ced"}, + {file = "more-itertools-8.8.0.tar.gz", hash = "sha256:83f0308e05477c68f56ea3a888172c78ed5d5b3c282addb67508e7ba6c8f813a"}, + {file = "more_itertools-8.8.0-py3-none-any.whl", hash = "sha256:2cf89ec599962f2ddc4d568a05defc40e0a587fbc10d5989713638864c36be4d"}, ] packaging = [ {file = "packaging-20.9-py2.py3-none-any.whl", hash = "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"}, {file = "packaging-20.9.tar.gz", hash = "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5"}, ] +parmap = [ + {file = "parmap-1.5.2-py2.py3-none-any.whl", hash = "sha256:1724d30b66c05ce462990fb667e421c27b65331599738dd11e899e4758a88d76"}, + {file = "parmap-1.5.2.tar.gz", hash = "sha256:a15f33fa0e2a454f29aecfdd5a825c780b7d26d876eb6093476ddf5c47c8f9b9"}, +] parso = [ {file = "parso-0.7.1-py2.py3-none-any.whl", hash = "sha256:97218d9159b2520ff45eb78028ba8b50d2bc61dcc062a9682666f2dc4bd331ea"}, {file = "parso-0.7.1.tar.gz", hash = "sha256:caba44724b994a8a5e086460bb212abc5a8bc46951bf4a9a1210745953622eb9"}, ] pathlib2 = [ - {file = "pathlib2-2.3.5-py2.py3-none-any.whl", hash = "sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db"}, - {file = "pathlib2-2.3.5.tar.gz", hash = "sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868"}, + {file = "pathlib2-2.3.6-py2.py3-none-any.whl", hash = "sha256:3a130b266b3a36134dcc79c17b3c7ac9634f083825ca6ea9d8f557ee6195c9c8"}, + {file = "pathlib2-2.3.6.tar.gz", hash = "sha256:7d8bcb5555003cdf4a8d2872c538faa3a0f5d20630cb360e518ca3b981795e5f"}, ] pep8 = [ {file = "pep8-1.7.1-py2.py3-none-any.whl", hash = "sha256:b22cfae5db09833bb9bd7c8463b53e1a9c9b39f12e304a8d0bba729c501827ee"}, @@ -1251,6 +1298,10 @@ pexpect = [ {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"}, ] +pgcopy = [ + {file = "pgcopy-1.5.0-py2.py3-none-any.whl", hash = "sha256:4b0451a4dbbd4c74ed148c2f831df10be2e2368ee0f62115c24f5a2f76ed2528"}, + {file = "pgcopy-1.5.0.tar.gz", hash = "sha256:a616af868720cbb805ef92922d44ed4044e9e813a9a4666763c630b0dab8b4fb"}, +] pickleshare = [ {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, @@ -1259,9 +1310,6 @@ pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] -progress = [ - {file = "progress-1.5.tar.gz", hash = "sha256:69ecedd1d1bbe71bf6313d88d1e6c4d2957b7f1d4f71312c211257f7dae64372"}, -] prompt-toolkit = [ {file = "prompt_toolkit-1.0.18-py2-none-any.whl", hash = "sha256:f7eec66105baf40eda9ab026cd8b2e251337eea8d111196695d82e0c5f0af852"}, {file = "prompt_toolkit-1.0.18-py3-none-any.whl", hash = "sha256:37925b37a4af1f6448c76b7606e0285f79f434ad246dda007a27411cca730c6d"}, @@ -1270,6 +1318,23 @@ prompt-toolkit = [ {file = "prompt_toolkit-2.0.10-py3-none-any.whl", hash = "sha256:46642344ce457641f28fc9d1c9ca939b63dadf8df128b86f1b9860e59c73a5e4"}, {file = "prompt_toolkit-2.0.10.tar.gz", hash = "sha256:f15af68f66e664eaa559d4ac8a928111eebd5feda0c11738b5998045224829db"}, ] +psycopg2 = [ + {file = "psycopg2-2.8.6-cp27-cp27m-win32.whl", hash = "sha256:068115e13c70dc5982dfc00c5d70437fe37c014c808acce119b5448361c03725"}, + {file = "psycopg2-2.8.6-cp27-cp27m-win_amd64.whl", hash = "sha256:d160744652e81c80627a909a0e808f3c6653a40af435744de037e3172cf277f5"}, + {file = "psycopg2-2.8.6-cp34-cp34m-win32.whl", hash = "sha256:b8cae8b2f022efa1f011cc753adb9cbadfa5a184431d09b273fb49b4167561ad"}, + {file = "psycopg2-2.8.6-cp34-cp34m-win_amd64.whl", hash = "sha256:f22ea9b67aea4f4a1718300908a2fb62b3e4276cf00bd829a97ab5894af42ea3"}, + {file = "psycopg2-2.8.6-cp35-cp35m-win32.whl", hash = "sha256:26e7fd115a6db75267b325de0fba089b911a4a12ebd3d0b5e7acb7028bc46821"}, + {file = "psycopg2-2.8.6-cp35-cp35m-win_amd64.whl", hash = "sha256:00195b5f6832dbf2876b8bf77f12bdce648224c89c880719c745b90515233301"}, + {file = "psycopg2-2.8.6-cp36-cp36m-win32.whl", hash = "sha256:a49833abfdede8985ba3f3ec641f771cca215479f41523e99dace96d5b8cce2a"}, + {file = "psycopg2-2.8.6-cp36-cp36m-win_amd64.whl", hash = "sha256:f974c96fca34ae9e4f49839ba6b78addf0346777b46c4da27a7bf54f48d3057d"}, + {file = "psycopg2-2.8.6-cp37-cp37m-win32.whl", hash = "sha256:6a3d9efb6f36f1fe6aa8dbb5af55e067db802502c55a9defa47c5a1dad41df84"}, + {file = "psycopg2-2.8.6-cp37-cp37m-win_amd64.whl", hash = "sha256:56fee7f818d032f802b8eed81ef0c1232b8b42390df189cab9cfa87573fe52c5"}, + {file = "psycopg2-2.8.6-cp38-cp38-win32.whl", hash = "sha256:ad2fe8a37be669082e61fb001c185ffb58867fdbb3e7a6b0b0d2ffe232353a3e"}, + {file = "psycopg2-2.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:56007a226b8e95aa980ada7abdea6b40b75ce62a433bd27cec7a8178d57f4051"}, + {file = "psycopg2-2.8.6-cp39-cp39-win32.whl", hash = "sha256:2c93d4d16933fea5bbacbe1aaf8fa8c1348740b2e50b3735d1b0bf8154cbf0f3"}, + {file = "psycopg2-2.8.6-cp39-cp39-win_amd64.whl", hash = "sha256:d5062ae50b222da28253059880a871dc87e099c25cb68acf613d9d227413d6f7"}, + {file = "psycopg2-2.8.6.tar.gz", hash = "sha256:fb23f6c71107c37fd667cb4ea363ddeb936b348bbd6449278eb92c189699f543"}, +] psycopg2-binary = [ {file = "psycopg2-binary-2.8.6.tar.gz", hash = "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0"}, {file = "psycopg2_binary-2.8.6-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:d14b140a4439d816e3b1229a4a525df917d6ea22a0771a2a78332273fd9528a4"}, @@ -1339,8 +1404,8 @@ pytest-cache = [ {file = "pytest-cache-1.0.tar.gz", hash = "sha256:be7468edd4d3d83f1e844959fd6e3fd28e77a481440a7118d430130ea31b07a9"}, ] pytest-cov = [ - {file = "pytest-cov-2.11.1.tar.gz", hash = "sha256:359952d9d39b9f822d9d29324483e7ba04a3a17dd7d05aa6beb7ea01e359e5f7"}, - {file = "pytest_cov-2.11.1-py2.py3-none-any.whl", hash = "sha256:bdb9fdb0b85a7cc825269a4c56b48ccaa5c7e365054b6038772c32ddcdc969da"}, + {file = "pytest-cov-2.12.1.tar.gz", hash = "sha256:261ceeb8c227b726249b376b8526b600f38667ee314f910353fa318caa01f4d7"}, + {file = "pytest_cov-2.12.1-py2.py3-none-any.whl", hash = "sha256:261bb9e47e65bd099c89c3edf92972865210c36813f80ede5277dceb77a4a62a"}, ] pytest-pep8 = [ {file = "pytest-pep8-1.0.6.tar.gz", hash = "sha256:032ef7e5fa3ac30f4458c73e05bb67b0f036a8a5cb418a534b3170f89f120318"}, @@ -1396,8 +1461,8 @@ simplegeneric = [ {file = "simplegeneric-0.8.1.zip", hash = "sha256:dc972e06094b9af5b855b3df4a646395e43d1c9d0d39ed345b7393560d0b9173"}, ] six = [ - {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, - {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] snowballstemmer = [ {file = "snowballstemmer-2.1.0-py2.py3-none-any.whl", hash = "sha256:b51b447bea85f9968c13b650126a888aabd4cb4463fca868ec596826325dedc2"}, @@ -1419,6 +1484,14 @@ text-unidecode = [ {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"}, {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, ] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +tqdm = [ + {file = "tqdm-4.61.2-py2.py3-none-any.whl", hash = "sha256:5aa445ea0ad8b16d82b15ab342de6b195a722d75fc1ef9934a46bba6feafbc64"}, + {file = "tqdm-4.61.2.tar.gz", hash = "sha256:8bb94db0d4468fea27d004a0f1d1c02da3cdedc00fe491c0de986b76a04d6b0a"}, +] traitlets = [ {file = "traitlets-4.3.3-py2.py3-none-any.whl", hash = "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44"}, {file = "traitlets-4.3.3.tar.gz", hash = "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7"}, @@ -1429,8 +1502,8 @@ typing = [ {file = "typing-3.10.0.0.tar.gz", hash = "sha256:13b4ad211f54ddbf93e5901a9967b1e07720c1d1b78d596ac6a439641aa1b130"}, ] urllib3 = [ - {file = "urllib3-1.26.4-py2.py3-none-any.whl", hash = "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df"}, - {file = "urllib3-1.26.4.tar.gz", hash = "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"}, + {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, + {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, ] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, diff --git a/pyproject.toml b/pyproject.toml index ec91a3c..ce1844f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "postgresql-anonymizer" -version = "0.5.0" +version = "0.6.0" description = "Commandline tool to anonymize PostgreSQL databases" authors = [ "Henning Kage " @@ -16,10 +16,11 @@ Faker = [ { version = "^3.0.0", python = "~2.7"}, { version = "^4.9.0", python = "^3.5"} ] -progress = "^1.5" psycopg2-binary = "^2.8.4" pyyaml = "^5.2" -six = "^1.13.0" +parmap = "^1.5.2" +tqdm = "^4.61.1" +pgcopy = "^1.5.0" [tool.poetry.dev-dependencies] flake8 = "^3.7.9" diff --git a/sample_schema.yml b/sample_schema.yml index 25990ad..067335e 100644 --- a/sample_schema.yml +++ b/sample_schema.yml @@ -14,6 +14,11 @@ tables: provider: name: md5 append: "@localhost" + - phone: + format: '+65-{pga_value}-55' + provider: + name: md5 + as_number: True excludes: - email: - "\\S[^@]*@example\\.com" diff --git a/tests/schemes/valid_schema.yml b/tests/schemes/valid_schema.yml new file mode 100644 index 0000000..25990ad --- /dev/null +++ b/tests/schemes/valid_schema.yml @@ -0,0 +1,22 @@ +tables: + - auth_user: + primary_key: id + chunk_size: 5000 + fields: + - first_name: + provider: + name: fake.first_name + - last_name: + provider: + name: set + value: "Bar" + - email: + provider: + name: md5 + append: "@localhost" + excludes: + - email: + - "\\S[^@]*@example\\.com" + +truncate: + - django_session diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..f393cf0 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,80 @@ +from tests.utils import quote_ident +from mock import call, patch, Mock +from pganonymizer.cli import get_arg_parser, main +import pytest +import shlex +from argparse import Namespace + + +class TestCli: + @patch('psycopg2.extensions.quote_ident', side_effect=quote_ident) + @patch('pganonymizer.utils.psycopg2.connect') + @patch('pganonymizer.utils.subprocess') + @pytest.mark.parametrize('cli_args, expected, expected_executes, commit_calls, call_dump', [ + ['--host localhost --port 5432 --user root --password my-cool-password --dbname db --schema ./tests/schemes/valid_schema.yml -v --init-sql "set work_mem=\'1GB\'"', # noqa + Namespace(verbose=1, list_providers=False, schema='./tests/schemes/valid_schema.yml', dbname='db', user='root', + password='my-cool-password', host='localhost', port='5432', dry_run=False, dump_file=None, init_sql="set work_mem='1GB'"), # noqa + [call("set work_mem='1GB'"), + call('TRUNCATE TABLE "django_session"'), + call('SELECT COUNT(*) FROM "auth_user"'), + call('SELECT "id", "first_name", "last_name", "email" FROM "auth_user"'), + call( + 'CREATE TEMP TABLE "tmp_auth_user" AS SELECT "id", "first_name", "last_name", "email"\n FROM "auth_user" WITH NO DATA'), # noqa + call('CREATE INDEX ON "tmp_auth_user" ("id")'), + call('UPDATE "auth_user" t SET "first_name" = s."first_name", "last_name" = s."last_name", "email" = s."email" FROM "tmp_auth_user" s WHERE t."id" = s."id"') # noqa + ], + 1, + [] + ], + ['--dry-run --host localhost --port 5432 --user root --password my-cool-password --dbname db --schema ./tests/schemes/valid_schema.yml -v --init-sql "set work_mem=\'1GB\'"', # noqa + Namespace(verbose=1, list_providers=False, schema='./tests/schemes/valid_schema.yml', dbname='db', user='root', + password='my-cool-password', host='localhost', port='5432', dry_run=True, dump_file=None, init_sql="set work_mem='1GB'"), # noqa + [call("set work_mem='1GB'"), + call('TRUNCATE TABLE "django_session"'), + call('SELECT "id", "first_name", "last_name", "email" FROM "auth_user" LIMIT 100'), + call('CREATE TEMP TABLE "tmp_auth_user" AS SELECT "id", "first_name", "last_name", "email"\n FROM "auth_user" WITH NO DATA'), # noqa + call('CREATE INDEX ON "tmp_auth_user" ("id")'), + call('UPDATE "auth_user" t SET "first_name" = s."first_name", "last_name" = s."last_name", "email" = s."email" FROM "tmp_auth_user" s WHERE t."id" = s."id"') # noqa + ], + 0, [] + ], + ['--dump-file ./dump.sql --host localhost --port 5432 --user root --password my-cool-password --dbname db --schema ./tests/schemes/valid_schema.yml -v --init-sql "set work_mem=\'1GB\'"', # noqa + Namespace(verbose=1, list_providers=False, schema='./tests/schemes/valid_schema.yml', dbname='db', user='root', + password='my-cool-password', host='localhost', port='5432', dry_run=False, dump_file='./dump.sql', init_sql="set work_mem='1GB'"), # noqa + [ + call("set work_mem='1GB'"), + call('TRUNCATE TABLE "django_session"'), + call('SELECT COUNT(*) FROM "auth_user"'), + call('SELECT "id", "first_name", "last_name", "email" FROM "auth_user"'), + call( + 'CREATE TEMP TABLE "tmp_auth_user" AS SELECT "id", "first_name", "last_name", "email"\n FROM "auth_user" WITH NO DATA'), # noqa + call('CREATE INDEX ON "tmp_auth_user" ("id")'), + call('UPDATE "auth_user" t SET "first_name" = s."first_name", "last_name" = s."last_name", "email" = s."email" FROM "tmp_auth_user" s WHERE t."id" = s."id"') # noqa + ], + 1, + [call('pg_dump -p -Fc -Z 9 -d db -U root -h localhost -p 5432 -f ./dump.sql', shell=True)] + ], + + ['--list-providers', + Namespace(verbose=None, list_providers=True, schema='schema.yml', dbname=None, user=None, + password='', host='localhost', port='5432', dry_run=False, dump_file=None, init_sql=False), + [], 0, [] + ] + ]) + def test_cli_args(self, subprocess, patched_connect, quote_ident, cli_args, expected, expected_executes, commit_calls, call_dump): # noqa + arg_parser = get_arg_parser() + parsed_args = arg_parser.parse_args(shlex.split(cli_args)) + assert parsed_args == expected + mock_cursor = Mock() + mock_cursor.fetchone.return_value = [0] + mock_cursor.fetchmany.return_value = None + + connection = Mock() + connection.cursor.return_value = mock_cursor + + patched_connect.return_value = connection + main(parsed_args) + assert mock_cursor.execute.call_args_list == expected_executes + assert connection.commit.call_count == commit_calls + + assert subprocess.run.call_args_list == call_dump diff --git a/tests/test_providers.py b/tests/test_providers.py index 8a8216b..4ca757e 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -1,4 +1,5 @@ import operator +import uuid import pytest import six @@ -63,6 +64,17 @@ def test_alter_value(self): assert isinstance(value, six.string_types) assert len(value) == 32 + def test_as_number(self): + provider = providers.MD5Provider(as_number=True) + value = provider.alter_value('foo') + assert isinstance(value, six.integer_types) + assert value == 985560 + + provider = providers.MD5Provider(as_number=True, as_number_length=8) + value = provider.alter_value('foobarbazadasd') + assert isinstance(value, six.integer_types) + assert value == 45684001 + class TestSetProvider: @@ -73,3 +85,21 @@ class TestSetProvider: def test_alter_value(self, kwargs, expected): provider = providers.SetProvider(**kwargs) assert provider.alter_value('Foo') == expected + + +class TestUUID4Provider: + @pytest.mark.parametrize('kwargs, expected', [ + ({'value': None}, None), + ({'value': 'Bar'}, 'Bar') + ]) + def test_alter_value(self, kwargs, expected): + provider = providers.UUID4Provider(**kwargs) + assert type(provider.alter_value('Foo')) == uuid.UUID + + +class TestInvalidProvider: + def test(self): + with pytest.raises(exceptions.InvalidProvider, + match="Could not find provider with id asdfnassdladladjasldasdklj"): + provider = providers.get_provider({'name': 'asdfnassdladladjasldasdklj', 'value': 'Foo'}) + provider.alter_value('Foo') diff --git a/tests/test_utils.py b/tests/test_utils.py index 9b75635..443bae0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,12 @@ -import pytest +from collections import OrderedDict, namedtuple +import math +from tests.utils import quote_ident from mock import ANY, Mock, call, patch -from pganonymizer.utils import get_connection, import_data, truncate_tables +import pytest + +from pganonymizer.utils import anonymize_tables, build_and_then_import_data, get_column_values, \ + get_connection, import_data, truncate_tables class TestGetConnection: @@ -20,19 +25,19 @@ def test(self, mock_connect): class TestTruncateTables: - - @pytest.mark.parametrize('tables, expected_sql', [ - [('table_a', 'table_b'), 'TRUNCATE TABLE table_a, table_b;'], + @patch('psycopg2.extensions.quote_ident', side_effect=quote_ident) + @pytest.mark.parametrize('tables, expected', [ + [('table_a', 'table_b', 'CAPS_TABLe'), 'TRUNCATE TABLE "table_a", "table_b", "CAPS_TABLe"'], [(), None], ]) - def test(self, tables, expected_sql): + def test(self, quote_ident, tables, expected): mock_cursor = Mock() connection = Mock() connection.cursor.return_value = mock_cursor truncate_tables(connection, tables) if tables: connection.cursor.assert_called_once() - assert mock_cursor.execute.call_args_list == [call(expected_sql)] + assert mock_cursor.execute.call_args_list == [call(expected)] mock_cursor.close.assert_called_once() else: connection.cursor.assert_not_called() @@ -41,21 +46,183 @@ def test(self, tables, expected_sql): class TestImportData: + @patch('psycopg2.extensions.quote_ident', side_effect=quote_ident) + @patch('pgcopy.copy.util') + @patch('pgcopy.copy.inspect') + @pytest.mark.parametrize('tmp_table, cols, data', [ + ['public.src_tbl', ('id', 'location'), [ + OrderedDict([("id", 0), ('location', 'Jerusalem')]), + OrderedDict([("id", 1), ('location', 'New York')]), + OrderedDict([("id", 2), ('location', 'Moscow')]), + ]] + ]) + def test(self, inspect, util, quote_ident, tmp_table, cols, data): + mock_cursor = Mock() + + connection = Mock() + connection.cursor.return_value = mock_cursor + connection.encoding = 'UTF8' + Record = namedtuple("Record", "attname,type_category,type_name,type_mod,not_null,typelem") + + inspect.get_types.return_value = { + 'id': Record(attname='id', type_category='N', type_name='int8', type_mod=-1, not_null=False, typelem=0), + 'location': Record(attname='location', type_category='S', type_name='varchar', type_mod=259, + not_null=False, typelem=0) + } + + import_data(connection, tmp_table, cols, data) - @pytest.mark.parametrize('source_table, table_columns, primary_key, expected_tbl_name, expected_columns', [ - ['src_tbl', ['id', 'COL_1'], 'id', 'tmp_src_tbl', ['"id"', '"COL_1"']] + # assert connection.cursor.call_count == mock_cursor.close.call_count + + mock_cursor.copy_expert.assert_called_once() + expected = [call('COPY "public"."src_tbl" ("id", "location") FROM STDIN WITH BINARY', ANY)] + assert mock_cursor.copy_expert.call_args_list == expected + + @ patch('pganonymizer.utils.CopyManager') + @ patch('psycopg2.extensions.quote_ident', side_effect=quote_ident) + def test_anonymize_tables(self, quote_ident, copy_manager): + mock_cursor = Mock() + mock_cursor.fetchone.return_value = [2] + mock_cursor.fetchmany.side_effect = [ + [ + OrderedDict([("first_name", None), + ("json_column", None) + ]), + OrderedDict([("first_name", "exclude me"), + ("json_column", {"field1": "foo"}) + ]), + OrderedDict([("first_name", "John Doe"), + ("json_column", {"field1": "foo"}) + ]), + OrderedDict([("first_name", "John Doe"), + ("json_column", {"field2": "bar"}) + ]) + ] + ] + cmm = Mock() + copy_manager.return_value = cmm + copy_manager.copy.return_value = [] + + connection = Mock() + connection.cursor.return_value = mock_cursor + definitions = [] + anonymize_tables(connection, definitions, verbose=True) + + assert connection.cursor.call_count == 0 + assert mock_cursor.close.call_count == 0 + assert copy_manager.copy.call_count == 0 + + definitions = [ + { + "auth_user": { + "primary_key": "id", + "chunk_size": 5000, + "excludes": [ + {'first_name': ['exclude']} + ], + "fields": [ + { + "first_name": { + "provider": { + "name": "set", + "value": "dummy name" + }, + "append": "append-me" + } + }, + { + "json_column.field1": { + "provider": { + "name": "set", + "value": "dummy json field1" + } + } + }, + { + "json_column.field2": { + "provider": { + "name": "set", + "value": "dummy json field2" + } + } + }, + ], + 'search': 'first_name == "John"' + } + } + ] + + anonymize_tables(connection, definitions, verbose=True) + assert connection.cursor.call_count == mock_cursor.close.call_count + assert copy_manager.call_args_list == [call(connection, 'tmp_auth_user', ['id', 'first_name', 'json_column'])] + assert cmm.copy.call_count == 1 + assert cmm.copy.call_args_list == [call([['dummy nameappend-me', b'{"field1": "dummy json field1"}'], + ['dummy nameappend-me', b'{"field2": "dummy json field2"}']])] + + +class TestBuildAndThenImport: + @ patch('psycopg2.extensions.quote_ident', side_effect=quote_ident) + @ patch('pganonymizer.utils.CopyManager') + @ pytest.mark.parametrize('table, primary_key, columns, total_count, chunk_size', [ + ['src_tbl', 'id', [{'col1': {'provider': {'name': 'md5'}}}, + {'COL2': {'provider': {'name': 'md5'}}}], 10, 3] ]) - def test(self, source_table, table_columns, primary_key, expected_tbl_name, expected_columns): + def test(self, quote_ident, copy_manager, table, primary_key, columns, total_count, chunk_size): + fake_record = dict.fromkeys([list(definition.keys())[0] for definition in columns], "") + records = [ + [fake_record for row in range(0, chunk_size)] for x in range(0, int(math.ceil(total_count / chunk_size))) + ] + mock_cursor = Mock() + mock_cursor.fetchmany.side_effect = records + mock_cursor.fetchone.return_value = [{}] connection = Mock() connection.cursor.return_value = mock_cursor - import_data(connection, {}, source_table, table_columns, primary_key, []) + build_and_then_import_data(connection, table, primary_key, columns, None, None, total_count, chunk_size) - assert connection.cursor.call_count == 2 - assert mock_cursor.close.call_count == 2 + expected_execute_calls = [call('SELECT "id", "col1", "COL2" FROM "src_tbl"'), + call( + 'CREATE TEMP TABLE "tmp_src_tbl" AS SELECT "id", "col1", "COL2"\n FROM "src_tbl" WITH NO DATA'), # noqa + call('CREATE INDEX ON "tmp_src_tbl" ("id")'), + call('UPDATE "src_tbl" t SET "col1" = s."col1", "COL2" = s."COL2" FROM "tmp_src_tbl" s WHERE t."id" = s."id"')] # noqa + assert mock_cursor.execute.call_args_list == expected_execute_calls - mock_cursor.copy_from.assert_called_once() - expected = [call(ANY, expected_tbl_name, columns=expected_columns, null=ANY, sep=ANY)] - assert mock_cursor.copy_from.call_args_list == expected + @patch('pganonymizer.utils.CopyManager') + def test_column_format(self, copy_manager): + columns = [ + { + "first_name": { + "format": "hello-{pga_value}-world", + "provider": { + "name": "set", + "value": "dummy name" + } + } + }, + { + "phone": { + "format": "+65-{pga_value}", + "provider": { + "name": "md5", + "as_number": True + } + } + }, + { + "templated": { + "format": "{pga_value}-{phone}-{first_name}", + "provider": { + "name": "set", + "value": "hello" + } + } + } + ] + row = OrderedDict([("first_name", "John Doe"), ("phone", "2354223432"), ("templated", "")]) + result = get_column_values(row, columns) + expected = {'first_name': 'hello-dummy name-world', + 'phone': '+65-91042872', + 'templated': "hello-+65-91042872-hello-dummy name-world"} + assert result == expected diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..cd9372d --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,5 @@ +# Don't use this function anywhere else that tests +def quote_ident(a, real_db_connection): + # quote_ident method implementation for test cases, + # it's used since original implementation requres a proper connection, and not mock + return '"{}"'.format(a)