From f64e14e713e57572c96970933d95981fdf34bcd8 Mon Sep 17 00:00:00 2001 From: Darek Date: Wed, 15 Apr 2026 17:34:45 -0500 Subject: [PATCH] Replace migrate_to_mariadb.py with sbin/setup_mariadb.py Covers all 5 models/tables, prompts for credentials, snapshots SQLite, and prints env/service config at the end. --- MIGRATION.md | 44 ++++--- migrate_to_mariadb.py | 129 ------------------ sbin/setup_mariadb.py | 296 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 324 insertions(+), 145 deletions(-) delete mode 100644 migrate_to_mariadb.py create mode 100755 sbin/setup_mariadb.py diff --git a/MIGRATION.md b/MIGRATION.md index d3cdf27..36208fe 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -217,26 +217,32 @@ flask db upgrade ## 5. Run the migration script -The `migrate_to_mariadb.py` script reads every record from SQLite and inserts -it into MariaDB using the SQLAlchemy ORM — no raw SQL, no CSV exports. +`sbin/setup_mariadb.py` prompts interactively for credentials (or reads them +from environment variables), snapshots the SQLite database, creates all tables +on MariaDB, and migrates every record using the SQLAlchemy ORM. ```bash +# Interactive — prompts for host, port, user, password, database +python sbin/setup_mariadb.py + +# Non-interactive — supply individual env vars +MARIADB_HOST=localhost MARIADB_PORT=3306 \ + MARIADB_USER=battuser MARIADB_PASSWORD=strongpassword \ + MARIADB_DATABASE=batteries \ + python sbin/setup_mariadb.py + +# Non-interactive — supply a full URL MARIADB_URL='mysql+pymysql://battuser:strongpassword@localhost/batteries?charset=utf8mb4' \ - python migrate_to_mariadb.py -``` - -Or pass the URL as a positional argument: - -```bash -python migrate_to_mariadb.py 'mysql+pymysql://battuser:strongpassword@localhost/batteries?charset=utf8mb4' + python sbin/setup_mariadb.py ``` The script: -1. Creates all tables on MariaDB if they don't exist -2. Inserts all Device records (preserving primary keys) -3. Inserts all Battery records (preserving primary keys and foreign keys) +1. Snapshots `batteries.db` → `batteries.db.YYYY-MM-DD.snapshot` +2. Creates all tables on MariaDB if they don't exist +3. Migrates Device, Battery, CapacityTest, ChargeLog, and BatteryPctLog records (preserving primary keys and foreign keys) 4. Resets `AUTO_INCREMENT` counters past the highest migrated ID 5. Prints a verification table comparing row counts +6. Prints the exact `DATABASE_URL` and systemd service line to configure --- @@ -246,10 +252,13 @@ The migration script prints a summary: ``` === Verification === -Table SQLite MariaDB OK? --------------------------------------- -device 5 5 OK -battery 40 40 OK +Table SQLite MariaDB OK? +----------------------------------------------- +device 5 5 OK +battery 40 40 OK +capacity_test 12 12 OK +charge_log 87 87 OK +battery_pct_log 320 320 OK Migration complete. All row counts match. ``` @@ -262,6 +271,9 @@ You can also verify manually in the MariaDB shell: ```sql SELECT COUNT(*) FROM device; SELECT COUNT(*) FROM battery; +SELECT COUNT(*) FROM capacity_test; +SELECT COUNT(*) FROM charge_log; +SELECT COUNT(*) FROM battery_pct_log; ``` --- diff --git a/migrate_to_mariadb.py b/migrate_to_mariadb.py deleted file mode 100644 index 74956dd..0000000 --- a/migrate_to_mariadb.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -Migrate data from SQLite to MariaDB using SQLAlchemy ORM only — no raw SQL -except for the AUTO_INCREMENT reset which requires a DDL statement. - -Usage: - MARIADB_URL='mysql+pymysql://user:pass@host/dbname?charset=utf8mb4' \\ - python migrate_to_mariadb.py - -Or pass the URL as a CLI argument: - python migrate_to_mariadb.py 'mysql+pymysql://user:pass@host/dbname?charset=utf8mb4' -""" - -import sys - -from sqlalchemy import create_engine, text -from sqlalchemy.orm import sessionmaker - -import config -from models import Base, Battery, Device - - -def migrate(mariadb_url: str): - print("=== Battery Tracker: SQLite → MariaDB Migration ===\n") - - # -- Engines -- - sqlite_engine = create_engine(config.SQLALCHEMY_DATABASE_URI) - mariadb_engine = create_engine(mariadb_url, pool_pre_ping=True) - - SrcSession = sessionmaker(bind=sqlite_engine) - DstSession = sessionmaker(bind=mariadb_engine) - - src = SrcSession() - dst = DstSession() - - try: - # -- Create tables on MariaDB -- - print("Creating tables on MariaDB (if not exist)…") - Base.metadata.create_all(mariadb_engine) - - # -- Read source data -- - src_devices = src.query(Device).all() - src_batteries = src.query(Battery).all() - print(f"Source: {len(src_devices)} devices, {len(src_batteries)} batteries\n") - - # -- Migrate Devices first (batteries have FK → device) -- - print("Migrating devices…") - for d in src_devices: - new_d = Device( - id=d.id, - name=d.name, - battery_slots=d.battery_slots, - notes=d.notes, - ) - dst.add(new_d) - dst.flush() - - # -- Migrate Batteries -- - print("Migrating batteries…") - for b in src_batteries: - new_b = Battery( - id=b.id, - label=b.label, - brand=b.brand, - status=b.status, - device_id=b.device_id, - notes=b.notes, - ) - dst.add(new_b) - dst.flush() - - dst.commit() - print("Commit successful.\n") - - # -- Reset AUTO_INCREMENT so new rows don't collide -- - # This DDL is required; there is no ORM-level equivalent. - with mariadb_engine.connect() as conn: - max_device_id = max((d.id for d in src_devices), default=0) - max_battery_id = max((b.id for b in src_batteries), default=0) - conn.execute(text("ALTER TABLE device SET AUTO_INCREMENT = :v"), - {"v": max_device_id + 1}) - conn.execute(text("ALTER TABLE battery SET AUTO_INCREMENT = :v"), - {"v": max_battery_id + 1}) - conn.commit() - print("AUTO_INCREMENT counters reset.\n") - - # -- Verify counts -- - dst_device_count = dst.query(Device).count() - dst_battery_count = dst.query(Battery).count() - src_device_count = len(src_devices) - src_battery_count = len(src_batteries) - - print("=== Verification ===") - print(f"{'Table':<12} {'SQLite':>8} {'MariaDB':>9} {'OK?':>5}") - print("-" * 38) - - device_ok = src_device_count == dst_device_count - battery_ok = src_battery_count == dst_battery_count - - print(f"{'device':<12} {src_device_count:>8} {dst_device_count:>9} {'OK' if device_ok else 'MISMATCH':>5}") - print(f"{'battery':<12} {src_battery_count:>8} {dst_battery_count:>9} {'OK' if battery_ok else 'MISMATCH':>5}") - - if not (device_ok and battery_ok): - print("\nERROR: Row count mismatch. Do not decommission SQLite.") - sys.exit(1) - - print("\nMigration complete. All row counts match.") - - except Exception as exc: - dst.rollback() - print(f"\nERROR: {exc}") - raise - finally: - src.close() - dst.close() - - -if __name__ == "__main__": - if len(sys.argv) > 1: - url = sys.argv[1] - else: - import os - url = os.environ.get("MARIADB_URL") - - if not url: - print("ERROR: Provide MariaDB URL via MARIADB_URL env var or as a CLI argument.") - print(" Example: python migrate_to_mariadb.py 'mysql+pymysql://user:pass@host/db?charset=utf8mb4'") - sys.exit(1) - - migrate(url) diff --git a/sbin/setup_mariadb.py b/sbin/setup_mariadb.py new file mode 100755 index 0000000..5b087a6 --- /dev/null +++ b/sbin/setup_mariadb.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +""" +sbin/setup_mariadb.py +===================== +Interactive setup and migration script: creates all Battery Tracker tables on +MariaDB and populates them from the existing SQLite database. + +Run from the repository root: + python sbin/setup_mariadb.py + +Credentials are read from environment variables; missing ones are prompted: + MARIADB_HOST (default: localhost) + MARIADB_PORT (default: 3306) + MARIADB_USER + MARIADB_PASSWORD + MARIADB_DATABASE + +Or supply a full URL via MARIADB_URL to skip individual prompts entirely: + MARIADB_URL='mysql+pymysql://user:pass@host/db?charset=utf8mb4' \\ + python sbin/setup_mariadb.py +""" + +import getpass +import os +import shutil +import sys +from datetime import date +from pathlib import Path + +# Allow imports from the repo root regardless of where this script is invoked. +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker + +import config +from models import Base, Battery, BatteryPctLog, CapacityTest, ChargeLog, Device + + +# --------------------------------------------------------------------------- +# Credential helpers +# --------------------------------------------------------------------------- + +def _env_or_prompt(var: str, label: str, default: str | None = None) -> str: + val = os.environ.get(var, "").strip() + if val: + return val + suffix = f" [{default}]" if default else "" + answer = input(f" {label}{suffix}: ").strip() + return answer or (default or "") + + +def _env_or_prompt_secret(var: str, label: str) -> str: + val = os.environ.get(var, "").strip() + if val: + return val + return getpass.getpass(f" {label}: ") + + +def collect_credentials() -> str: + """Return a MariaDB SQLAlchemy URL, prompting for any missing pieces.""" + url = os.environ.get("MARIADB_URL", "").strip() + if url: + print(f" Using MARIADB_URL from environment.") + return url + + print("Enter MariaDB connection details (press Enter to accept defaults):\n") + host = _env_or_prompt("MARIADB_HOST", "Host", "localhost") + port = _env_or_prompt("MARIADB_PORT", "Port", "3306") + user = _env_or_prompt("MARIADB_USER", "User") + password = _env_or_prompt_secret("MARIADB_PASSWORD", "Password") + database = _env_or_prompt("MARIADB_DATABASE", "Database") + + if not user or not database: + print("\nERROR: MARIADB_USER and MARIADB_DATABASE are required.") + sys.exit(1) + + return f"mysql+pymysql://{user}:{password}@{host}:{port}/{database}?charset=utf8mb4" + + +# --------------------------------------------------------------------------- +# Snapshot +# --------------------------------------------------------------------------- + +def snapshot_sqlite() -> None: + """Copy batteries.db → batteries.db.YYYY-MM-DD.snapshot (in repo root).""" + src = REPO_ROOT / "batteries.db" + if not src.exists(): + print(" No batteries.db found — skipping snapshot (fresh install).") + return + dst = REPO_ROOT / f"batteries.db.{date.today().isoformat()}.snapshot" + shutil.copy2(src, dst) + print(f" Snapshot written: {dst.name}") + + +# --------------------------------------------------------------------------- +# Migration +# --------------------------------------------------------------------------- + +def migrate(mariadb_url: str) -> None: + print("\n=== Battery Tracker: SQLite → MariaDB Migration ===\n") + + # -- Engines -- + sqlite_engine = create_engine(config.SQLALCHEMY_DATABASE_URI) + mariadb_engine = create_engine(mariadb_url, pool_pre_ping=True) + + SrcSession = sessionmaker(bind=sqlite_engine) + DstSession = sessionmaker(bind=mariadb_engine) + + src = SrcSession() + dst = DstSession() + + try: + # -- Create tables -- + print("Creating tables on MariaDB (if not exist)…") + Base.metadata.create_all(mariadb_engine) + + # -- Read source data -- + src_devices = src.query(Device).all() + src_batteries = src.query(Battery).all() + src_cap_tests = src.query(CapacityTest).all() + src_charge_logs = src.query(ChargeLog).all() + src_pct_logs = src.query(BatteryPctLog).all() + + print( + f"Source: {len(src_devices)} devices, {len(src_batteries)} batteries, " + f"{len(src_cap_tests)} capacity tests, {len(src_charge_logs)} charge logs, " + f"{len(src_pct_logs)} pct logs\n" + ) + + # -- Devices (no FK dependencies) -- + print("Migrating devices…") + for d in src_devices: + dst.add(Device( + id=d.id, + name=d.name, + battery_slots=d.battery_slots, + device_type=d.device_type, + notes=d.notes, + ha_entity_id=d.ha_entity_id, + )) + dst.flush() + + # -- Batteries (FK → device) -- + print("Migrating batteries…") + for b in src_batteries: + dst.add(Battery( + id=b.id, + label=b.label, + brand=b.brand, + status=b.status, + device_id=b.device_id, + notes=b.notes, + size=b.size, + chemistry=b.chemistry, + capacity_mah=b.capacity_mah, + tested_capacity_mah=b.tested_capacity_mah, + tested_date=b.tested_date, + charge_cycles=b.charge_cycles, + purchase_date=b.purchase_date, + storage_location=b.storage_location, + battery_percentage=b.battery_percentage, + )) + dst.flush() + + # -- CapacityTest (FK → battery) -- + print("Migrating capacity tests…") + for ct in src_cap_tests: + dst.add(CapacityTest( + id=ct.id, + battery_id=ct.battery_id, + tested_capacity_mah=ct.tested_capacity_mah, + tested_date=ct.tested_date, + notes=ct.notes, + )) + dst.flush() + + # -- ChargeLog (FK → battery) -- + print("Migrating charge logs…") + for cl in src_charge_logs: + dst.add(ChargeLog( + id=cl.id, + battery_id=cl.battery_id, + charged_date=cl.charged_date, + increment_cycles=cl.increment_cycles, + notes=cl.notes, + )) + dst.flush() + + # -- BatteryPctLog (FK → battery) -- + print("Migrating battery pct logs…") + for pl in src_pct_logs: + dst.add(BatteryPctLog( + id=pl.id, + battery_id=pl.battery_id, + percentage=pl.percentage, + recorded_at=pl.recorded_at, + source=pl.source, + )) + dst.flush() + + dst.commit() + print("Commit successful.\n") + + # -- Reset AUTO_INCREMENT -- + table_max = { + "device": max((d.id for d in src_devices), default=0), + "battery": max((b.id for b in src_batteries), default=0), + "capacity_test": max((c.id for c in src_cap_tests), default=0), + "charge_log": max((c.id for c in src_charge_logs), default=0), + "battery_pct_log": max((p.id for p in src_pct_logs), default=0), + } + with mariadb_engine.connect() as conn: + for table, max_id in table_max.items(): + conn.execute( + text(f"ALTER TABLE {table} AUTO_INCREMENT = :v"), + {"v": max_id + 1}, + ) + conn.commit() + print("AUTO_INCREMENT counters reset.\n") + + # -- Verify counts -- + counts = { + "device": (len(src_devices), dst.query(Device).count()), + "battery": (len(src_batteries), dst.query(Battery).count()), + "capacity_test": (len(src_cap_tests), dst.query(CapacityTest).count()), + "charge_log": (len(src_charge_logs), dst.query(ChargeLog).count()), + "battery_pct_log": (len(src_pct_logs), dst.query(BatteryPctLog).count()), + } + + print("=== Verification ===") + print(f"{'Table':<20} {'SQLite':>8} {'MariaDB':>9} {'OK?':>6}") + print("-" * 47) + all_ok = True + for table, (src_n, dst_n) in counts.items(): + ok = src_n == dst_n + all_ok = all_ok and ok + print(f"{table:<20} {src_n:>8} {dst_n:>9} {'OK' if ok else 'MISMATCH':>6}") + + if not all_ok: + print("\nERROR: Row count mismatch. Do not decommission SQLite.") + sys.exit(1) + + print("\nMigration complete. All row counts match.") + + except Exception as exc: + dst.rollback() + print(f"\nERROR: {exc}") + raise + finally: + src.close() + dst.close() + + +# --------------------------------------------------------------------------- +# Post-migration instructions +# --------------------------------------------------------------------------- + +def print_next_steps(mariadb_url: str) -> None: + service_file = Path.home() / ".config/systemd/user/battery-tracker.service" + print(""" +=== Environment configuration === + +Add to your .env file (or export in shell before starting the app): + +""") + print(f" DATABASE_URL={mariadb_url}") + print(f""" +=== systemd service file change === + +Edit: {service_file} +In the [Service] section, add (or replace any existing DATABASE_URL line): + + Environment=DATABASE_URL={mariadb_url} + +Then reload and restart: + systemctl --user daemon-reload + systemctl --user restart battery-tracker +""") + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + print("=== Battery Tracker — MariaDB Setup ===\n") + + mariadb_url = collect_credentials() + + print("\nSnapshotting SQLite database…") + snapshot_sqlite() + + migrate(mariadb_url) + print_next_steps(mariadb_url)