diff --git a/.gitignore b/.gitignore index 68bc17f..367ce6b 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Test databases generated at the repo root +/*.db diff --git a/README.md b/README.md index ca50fda..41dd9dc 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ uvx sqlite2duckdb --force source.db target.db # overwrite target.db without as from sqlite2duckdb import sqlite_to_duckdb result = sqlite_to_duckdb("source.sqlite", "target.duckdb") -print(result.tables, result.elapsed) +print(result.tables, result.views, result.elapsed) ``` ## What is converted @@ -76,15 +76,19 @@ print(result.tables, result.elapsed) | | | |---|---| | Tables and data | ✅ | -| Primary keys, NOT NULL constraints, indexes | ✅ | -| UNIQUE, FOREIGN KEY and CHECK constraints | ❌ | -| Views | ❌ (silently dropped) | +| Primary keys, NOT NULL and UNIQUE constraints | ✅ | +| Indexes | ✅ | +| Views | ✅ best effort | +| FOREIGN KEY and CHECK constraints | ❌ | -## Todo +A view whose SQL uses something duckdb has no equivalent for (`MATCH`, `julianday()`) is +skipped with a warning instead of failing the conversion. + +FOREIGN KEY and CHECK are not copied, though not for lack of support: duckdb accepts and +enforces both in `CREATE TABLE`. It checks foreign keys row by row, so a self-referencing +table cannot be bulk loaded, and there is no `ALTER TABLE ADD CONSTRAINT` to add them once +the data is in. -- [ ] Custom type mapping -- [x] Primary keys, NOT NULL constraints and indexes -- [ ] Views, and UNIQUE / FOREIGN KEY / CHECK constraints ## Contributing diff --git a/pyproject.toml b/pyproject.toml index 05a973f..47043e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "sqlite2duckdb" -version = "0.4.0" +version = "0.5.0" authors = [{name="Sacha Schutz", email="sacha.schutz@pm.me"}] description = "A tool to convert sqlite database to duckdb database" readme = "README.md" diff --git a/sqlite2duckdb/sqlite_to_duckdb.py b/sqlite2duckdb/sqlite_to_duckdb.py index b1bcc82..e8a6339 100644 --- a/sqlite2duckdb/sqlite_to_duckdb.py +++ b/sqlite2duckdb/sqlite_to_duckdb.py @@ -20,6 +20,7 @@ class ConversionResult: tables: int elapsed: float """Wall clock duration of the conversion, in seconds.""" + views: int = 0 def _quote_identifier(name: str) -> str: @@ -96,6 +97,88 @@ def _copy_indexes(conn: duckdb.DuckDBPyConnection, sqlite_path: str) -> None: logger.warning("Could not recreate index %s: %s", name, error) +def _copy_unique_constraints( + conn: duckdb.DuckDBPyConnection, sqlite_path: str, table_names: list[str] +) -> None: + """Replay the UNIQUE constraints that sqlite records without any SQL. + + A column or table level UNIQUE becomes an autoindex whose sqlite_master row + has a NULL sql, so _copy_indexes cannot see it. Duckdb has no ALTER TABLE ADD + CONSTRAINT either, so a unique index is how the guarantee is carried over. + """ + + with contextlib.closing(sqlite3.connect(sqlite_path)) as source: + for table in table_names: + indexes = source.execute( + f"PRAGMA index_list({_quote_identifier(table)})" + ).fetchall() + + for _, index_name, unique, origin, _partial in indexes: + # 'c' indexes carry their own SQL and are handled by _copy_indexes, + # and 'pk' is already part of the table DDL. + if not unique or origin != "u": + continue + + columns = [ + row[2] + for row in source.execute( + f"PRAGMA index_info({_quote_identifier(index_name)})" + ).fetchall() + ] + if any(column is None for column in columns): + logger.warning( + "Skipping unique index %s: it is built on an expression", + index_name, + ) + continue + + targets = ", ".join(_quote_identifier(column) for column in columns) + try: + conn.sql( + f"CREATE UNIQUE INDEX {_quote_identifier(index_name)} " + f"ON {_quote_identifier(table)} ({targets})" + ) + except duckdb.Error as error: + logger.warning( + "Could not recreate unique index %s: %s", index_name, error + ) + + +def _copy_views(conn: duckdb.DuckDBPyConnection, sqlite_path: str) -> int: + """Recreate the source views, and return how many made it across.""" + + with contextlib.closing(sqlite3.connect(sqlite_path)) as source: + views = source.execute( + "SELECT name, sql FROM sqlite_master WHERE type = 'view' AND sql IS NOT NULL" + ).fetchall() + + pending = [(name, _brackets_to_quotes(sql)) for name, sql in views] + errors: dict[str, duckdb.Error] = {} + created = 0 + + # A view can sit on top of another one and sqlite_master does not guarantee + # dependency order, so keep retrying while a pass still makes progress. + while pending: + failed = [] + for name, statement in pending: + try: + conn.sql(statement) + except duckdb.Error as error: + errors[name] = error + failed.append((name, statement)) + else: + created += 1 + + if len(failed) == len(pending): + break + pending = failed + + for name, _ in pending: + logger.warning("Could not recreate view %s: %s", name, errors[name]) + + return created + + def _copy_tables( conn: duckdb.DuckDBPyConnection, tables: list[tuple[str, str]] ) -> None: @@ -118,9 +201,13 @@ def sqlite_to_duckdb( ) -> ConversionResult: """Copy a sqlite database into a new duckdb database. - Tables, data, primary keys, NOT NULL constraints and indexes are copied. - Views and UNIQUE / FOREIGN KEY / CHECK constraints are not: duckdb's sqlite - extension does not expose them on the attached database. + Tables, data, views, primary keys, NOT NULL and UNIQUE constraints and + indexes are copied. FOREIGN KEY and CHECK constraints are not: duckdb checks + foreign keys row by row, so a self-referencing table cannot be bulk loaded, + and there is no ALTER TABLE ADD CONSTRAINT to add them once the data is in. + + A view duckdb cannot bind is skipped with a warning rather than failing the + whole conversion. Raises FileNotFoundError if `sqlite_db` does not exist, and FileExistsError if `duck_db` already exists and `overwrite` is False. @@ -157,6 +244,10 @@ def sqlite_to_duckdb( _copy_tables(conn, tables) _copy_indexes(conn, sqlite_path) + _copy_unique_constraints(conn, sqlite_path, [name for name, _ in tables]) + views = _copy_views(conn, sqlite_path) + if views: + logger.info("%d view(s) copied", views) conn.sql("DETACH __other") except BaseException: @@ -171,4 +262,6 @@ def sqlite_to_duckdb( elapsed = time.perf_counter() - start_time logger.info("Done in %s !", _format_duration(elapsed)) - return ConversionResult(target=duck_path, tables=len(tables), elapsed=elapsed) + return ConversionResult( + target=duck_path, tables=len(tables), elapsed=elapsed, views=views + ) diff --git a/tests/conftest.py b/tests/conftest.py index d833073..c7dd3cb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,3 +49,13 @@ def duckdb_path(tmp_path): @pytest.fixture(scope="module") def bracket_index_sqlite(tmp_path_factory): return _module_db(tmp_path_factory, "bindex", utils.build_bracket_index_sqlite) + + +@pytest.fixture(scope="module") +def views_sqlite(tmp_path_factory): + return _module_db(tmp_path_factory, "views", utils.build_views_sqlite) + + +@pytest.fixture(scope="module") +def unique_sqlite(tmp_path_factory): + return _module_db(tmp_path_factory, "unique", utils.build_unique_sqlite) diff --git a/tests/test_convert.py b/tests/test_convert.py index 9dba590..8bd33e7 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -95,6 +95,11 @@ def test_bracket_quoted_view(bracket_sqlite, duckdb_path): (2, 7), ] + # The bracket quoted view built on it comes across too. + assert d_conn.sql( + 'SELECT * FROM "Order Subtotals" ORDER BY OrderID' + ).fetchall() == [(1, 5), (2, 7)] + def test_dotted_column_names(dotted_column_sqlite, duckdb_path): """Regression test for issue #4: a STRICT table with numeric-looking column diff --git a/tests/test_views_and_unique.py b/tests/test_views_and_unique.py new file mode 100644 index 0000000..78db10c --- /dev/null +++ b/tests/test_views_and_unique.py @@ -0,0 +1,87 @@ +"""Views and UNIQUE constraints, which duckdb's sqlite extension does not expose +on an attached database and which have to be read back from sqlite_master.""" + +import logging + +import duckdb +import pytest + +from sqlite2duckdb import sqlite_to_duckdb + + +def test_views_are_copied(views_sqlite, duckdb_path): + result = sqlite_to_duckdb(views_sqlite, duckdb_path) + + d_conn = duckdb.connect(str(duckdb_path)) + + assert d_conn.sql("SELECT * FROM by_region ORDER BY region").fetchall() == [ + ("north", 15), + ("south", 7), + ] + assert result.views == 3 + + +def test_view_built_on_another_view(views_sqlite, duckdb_path): + sqlite_to_duckdb(views_sqlite, duckdb_path) + + d_conn = duckdb.connect(str(duckdb_path)) + + assert d_conn.sql("SELECT * FROM big_regions").fetchall() == [("north",)] + + +def test_bracket_quoted_view_is_translated(views_sqlite, duckdb_path): + sqlite_to_duckdb(views_sqlite, duckdb_path) + + d_conn = duckdb.connect(str(duckdb_path)) + + assert d_conn.sql('SELECT COUNT(*) FROM "north sales"').fetchone() == (2,) + + +def test_view_duckdb_cannot_parse_is_skipped_with_a_warning( + views_sqlite, duckdb_path, caplog +): + with caplog.at_level(logging.WARNING, logger="sqlite2duckdb.sqlite_to_duckdb"): + sqlite_to_duckdb(views_sqlite, duckdb_path) + + assert "matched" in caplog.text + + d_conn = duckdb.connect(str(duckdb_path)) + views = { + row[0] + for row in d_conn.sql( + "SELECT view_name FROM duckdb_views() WHERE NOT internal" + ).fetchall() + } + + assert "matched" not in views + # The rest of the database must still be intact. + assert d_conn.sql("SELECT COUNT(*) FROM sales").fetchone() == (3,) + + +def test_unique_constraints_are_enforced(unique_sqlite, duckdb_path): + sqlite_to_duckdb(unique_sqlite, duckdb_path) + + d_conn = duckdb.connect(str(duckdb_path)) + + # Column level UNIQUE, recorded by sqlite as an autoindex with no SQL. + with pytest.raises(duckdb.ConstraintException): + d_conn.sql("INSERT INTO members VALUES (2, 'ada@example.com', 'x', 'y')") + + # Table level UNIQUE over two columns. + with pytest.raises(duckdb.ConstraintException): + d_conn.sql( + "INSERT INTO members VALUES (3, 'other@example.com', 'ada', 'lovelace')" + ) + + # An explicit CREATE UNIQUE INDEX, which does carry its SQL. + with pytest.raises(duckdb.ConstraintException): + d_conn.sql("INSERT INTO members VALUES (4, 'x@example.com', 'x', 'lovelace')") + + +def test_non_unique_rows_still_insert(unique_sqlite, duckdb_path): + sqlite_to_duckdb(unique_sqlite, duckdb_path) + + d_conn = duckdb.connect(str(duckdb_path)) + d_conn.sql("INSERT INTO members VALUES (5, 'grace@example.com', 'grace', 'hopper')") + + assert d_conn.sql("SELECT COUNT(*) FROM members").fetchone() == (2,) diff --git a/tests/utils.py b/tests/utils.py index 33a5639..827c33f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -200,3 +200,53 @@ def build_bracket_index_sqlite(path): conn.close() return path + + +def build_views_sqlite(path): + """Views, including one chained on another and one duckdb cannot bind.""" + + conn = sqlite3.connect(path) + conn.executescript( + """ + CREATE TABLE sales (id INTEGER, region TEXT, amount INTEGER); + INSERT INTO sales VALUES (1, 'north', 10), (2, 'north', 5), (3, 'south', 7); + + CREATE VIEW by_region AS + SELECT region, SUM(amount) AS total FROM sales GROUP BY region; + + CREATE VIEW big_regions AS + SELECT region FROM by_region WHERE total > 8; + + CREATE VIEW [north sales] AS SELECT * FROM sales WHERE [region] = 'north'; + + -- MATCH is sqlite only, and duckdb's parser rejects it outright. + CREATE VIEW matched AS SELECT * FROM sales WHERE region MATCH 'north'; + """ + ) + conn.commit() + conn.close() + + return path + + +def build_unique_sqlite(path): + """The three ways sqlite records uniqueness.""" + + conn = sqlite3.connect(path) + conn.executescript( + """ + CREATE TABLE members ( + id INTEGER PRIMARY KEY, + email TEXT UNIQUE, + first TEXT, + last TEXT, + UNIQUE (first, last) + ); + INSERT INTO members VALUES (1, 'ada@example.com', 'ada', 'lovelace'); + CREATE UNIQUE INDEX idx_members_last ON members (last); + """ + ) + conn.commit() + conn.close() + + return path diff --git a/uv.lock b/uv.lock index 68b580b..74c43d7 100644 --- a/uv.lock +++ b/uv.lock @@ -273,7 +273,7 @@ wheels = [ [[package]] name = "sqlite2duckdb" -version = "0.4.0" +version = "0.5.0" source = { editable = "." } dependencies = [ { name = "duckdb", version = "1.4.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },