{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "technicalSpecifications": {
    "productVersion": "0.4.10",
    "versionCheck": "LakeXpress --version prints e.g. 'LakeXpress 0.4.10 (build 2026-09-24T12:04:58Z)'; the build date is shown from 0.4.9",
    "architecture": {
      "type": "Command-line executable",
      "exportEngine": "FastBCP (bundled in engine/ in the trial edition; installed separately for the full edition)",
      "dataPath": "Source database -> FastBCP -> Parquet files on local disk or cloud storage -> optional publishing to a target platform",
      "stateStore": "LakeXpress DB: sync configurations, runs, jobs, files, columns, events, watermarks, publish history",
      "platforms": ["Linux x64", "Windows x64"],
      "distribution": "ZIP archive with the binary, README.txt, SBOM files (CycloneDX 1.5 JSON and XML, with SHA-256 checksums) and, in the trial edition, engine/FastBCP"
    },
    "commandLine": {
      "syntax": "LakeXpress [-h] [--version] [--no_banner] COMMAND [SUBCOMMAND] [OPTIONS]",
      "rule": "Only -h, --version and --no_banner may come before the command; all other options go after it",
      "commands": [
        { "command": "lxdb init", "purpose": "Create the LakeXpress tables in the LakeXpress DB; safe to run again" },
        { "command": "lxdb drop", "purpose": "Drop the LakeXpress tables and their data (dry run without --confirm)" },
        { "command": "lxdb truncate", "purpose": "Delete all data from the LakeXpress tables, keep the tables (dry run without --confirm)" },
        { "command": "lxdb locks", "purpose": "List tables locked by incremental syncs" },
        { "command": "lxdb release-locks", "purpose": "Release locks left by a crashed sync (--max_age_hours, --table_id; dry run without --confirm)" },
        { "command": "config create", "purpose": "Create a sync configuration" },
        { "command": "config list", "purpose": "List the sync configurations (--env_name)" },
        { "command": "config delete", "purpose": "Delete a sync configuration with its run history and watermarks; files and published tables are kept (dry run without --confirm)" },
        { "command": "sync", "purpose": "Run a sync configuration: export, then publish (--sync_id, --resume, --run_id, --fastbcp_dir_path)" },
        { "command": "sync[export]", "purpose": "Export only" },
        { "command": "sync[publish]", "purpose": "Publish the files of a previous run (--run_id; default latest completed run)" },
        { "command": "status", "purpose": "Show runs and their status (--sync_id or --run_id, --verbose)" },
        { "command": "cleanup", "purpose": "Remove runs stuck in running or failed (--older-than, --status, --dry-run)" },
        { "command": "run", "purpose": "Run a legacy YAML configuration file (-c / --config)" }
      ],
      "commonOptions": [
        { "option": "-a, --auth_file PATH", "description": "Credentials JSON file. Required by every command except run" },
        { "option": "--lxdb_auth_id ID", "description": "Credentials entry of the LakeXpress DB. Required by every command except run" },
        { "option": "--log_level LEVEL", "description": "DEBUG, INFO (default), WARNING or ERROR" },
        { "option": "--log_dir PATH", "description": "Directory for log files (default: current directory)" },
        { "option": "--no_progress", "description": "Disable the progress bar" },
        { "option": "--no_banner", "description": "Do not print the startup banner" },
        { "option": "--quiet_fbcp", "description": "Do not copy FastBCP log lines into the LakeXpress logs" },
        { "option": "--license TEXT", "description": "Licence text" },
        { "option": "--license_file PATH", "description": "Path to a licence file" }
      ],
      "configCreateOptions": {
        "required": ["--auth_file", "--lxdb_auth_id", "--source_db_auth_id", "--output_dir or --target_storage_id"],
        "identity": {
          "--sync_id ID": "1 to 64 letters, digits, _ or -; generated if omitted; fails if it already exists",
          "--env_name NAME": "Environment name (default: default)"
        },
        "source": {
          "--source_db_auth_id ID": "Credentials entry of the source database",
          "--source_db_name NAME": "Source database (default: info.database of the entry)",
          "--source_schema_name PATTERN": "Comma-separated schemas, SQL LIKE patterns; default all schemas"
        },
        "filtering": {
          "-i, --include PATTERN": "Only tables matching these LIKE patterns",
          "-e, --exclude PATTERN": "Skip tables matching these LIKE patterns",
          "--min_rows N": "Skip tables with fewer than N rows",
          "--max_rows N": "Skip tables with more than N rows"
        },
        "incremental": {
          "--incremental_table SPEC": "schema.table:column:type[:i|:e][@start][!strategy]; types date, datetime, int, bigint, auto; strategies append, upsert, delete_insert",
          "--incremental_safety_lag SECONDS": "Stay behind the current time (default 0)"
        },
        "output": {
          "--output_dir PATH": "Local directory (saved as absolute from 0.4.9)",
          "--target_storage_id ID": "Cloud storage entry (S3, GCS, Azure, OneLake)",
          "--sub_path PATH": "Folders between the base path and the schema folders"
        },
        "exportEngine": {
          "--fastbcp_dir_path PATH": "Folder containing FastBCP (not needed with the trial edition)",
          "--n_jobs N": "Tables exported in parallel (default 1)",
          "-p, --fastbcp_p N": "FastBCP processes within one large table (default 1)",
          "--large_table_threshold N": "Row count above which a table is exported in parallel (default 100000)",
          "--fastbcp_table_config CONFIG": "[schema.]table:method:key_column:degree, separated by ;. The degree is required",
          "--compression_type TYPE": "Zstd (default), Snappy, Gzip, Lz4, None"
        },
        "publishing": {
          "--publish_target ID": "Credentials entry of the target platform",
          "--publish_method METHOD": "external (default) or internal",
          "--publish_database_name NAME": "AWS Glue database or BigQuery dataset",
          "--publish_schema_pattern PATTERN": "Tokens {schema} {database} {date} {timestamp} {uuid} {subpath}; must not contain {table}",
          "--publish_table_pattern PATTERN": "Must contain {table}; default {table}",
          "--no_views": "Snowflake: no views over external tables",
          "--pk_constraints": "Add source primary keys to internal tables"
        },
        "metadata": {
          "--generate_metadata": "Write CDM manifest.json and .cdm.json files",
          "--manifest_name NAME": "Name of the CDM manifest"
        },
        "errorHandling": {
          "--error_action ACTION": "fail (default) or continue"
        }
      }
    },
    "credentialsFile": {
      "format": "JSON object; each top-level key is an entry id passed to --lxdb_auth_id, --source_db_auth_id, --target_storage_id or --publish_target",
      "entryShape": { "ds_type": "string", "auth_mode": "string", "info": "object" },
      "environmentVariables": "Any string value can be $env{VAR_NAME}; an unset variable stops the command",
      "example": {
        "lxdb_postgres": {
          "ds_type": "postgres",
          "auth_mode": "classic",
          "info": { "username": "$env{LX_LXDB_USER}", "password": "$env{LX_LXDB_PASSWORD}", "server": "localhost", "port": 5432, "database": "lakexpress_log" }
        },
        "source_postgres": {
          "ds_type": "postgres",
          "auth_mode": "classic",
          "info": { "username": "$env{LX_PG_USER}", "password": "$env{LX_PG_PASSWORD}", "server": "localhost", "port": 5432, "database": "production_db" }
        },
        "s3_01": {
          "ds_type": "s3",
          "auth_mode": "profile",
          "info": { "directory": "s3://my-bucket/lakexpress/exports", "profile": "my-aws-profile" }
        }
      },
      "dsTypes": {
        "sources": ["oracle", "postgres", "mssql", "mysql", "mariadb", "saphana", "teradata"],
        "lakeXpressDB": ["postgres", "mssql", "mysql", "sqlite", "duckdb"],
        "storage": ["s3", "gcs", "azure", "onelake"],
        "publishTargets": ["snowflake", "databricks", "fabric", "bigquery", "redshift", "motherduck", "aws_glue (or glue)", "DuckLake catalog: sqlite, duckdb, postgres or mysql"]
      },
      "security": "Restrict the file (chmod 600), never commit it, use a read-only source user"
    },
    "exportMethods": {
      "withoutKey": { "None": "All databases, sequential", "Ctid": "PostgreSQL", "Rowid": "Oracle", "Physloc": "SQL Server" },
      "withKey": {
        "Random": "Integer or bigint column",
        "DataDriven": "Column or expression with few distinct values, e.g. YEAR(l_shipdate)",
        "RangeId": "Numeric or date column, min/max ranges",
        "Ntile": "Numeric column, even chunks",
        "Timepartition": "(date_col,year[,month[,day]]) or (date_col,year,week); writes Hive-style folders"
      },
      "selection": "Automatic per table; override with --fastbcp_table_config or the partition_columns table"
    },
    "outputLayout": "<base>/[<sub_path>/]<schema>/<table>/part-00000.parquet",
    "exitCodes": {
      "0": "Success (also when no table matched the filters, and for dry runs without --confirm)",
      "1": "Unexpected error, or sync[publish] failed",
      "2": "Command-line error",
      "10": "Configuration error (credentials, unknown id, unset $env{}, sync not found, no output destination, FastBCP not found, invalid publish pattern, run already running...)",
      "12": "Connection failed",
      "13": "Licence invalid or expired",
      "21": "Credentials file not found (lxdb, config, status, cleanup)",
      "31": "lxdb init, drop or truncate failed",
      "40": "Some tables failed (export or publish): fix, then sync --resume"
    },
    "lakeXpressDBTables": {
      "export_syncs": "Sync configurations",
      "export_runs": "One row per run, with status and totals",
      "export_jobs": "One row per table and run: status, rows exported, files, FastBCP method and degree, timestamps",
      "export_files": "Each Parquet file: path, size, rows",
      "export_columns": "Column metadata per job",
      "export_log": "Events and errors",
      "export_steps": "Workflow steps",
      "partition_columns": "Persistent per-table FastBCP settings",
      "incremental_tables / incremental_watermarks / incremental_export_history": "Incremental configuration and watermarks",
      "publish_runs / publish_jobs": "Publishing history"
    },
    "logging": {
      "console": true,
      "logFile": "lx_YYYYMMDD_HHMMSS_<run_id>.log in --log_dir",
      "configurationCopy": "lx_YYYYMMDD_HHMMSS_<run_id>.yml next to the log, usable with run",
      "lineFormat": "[YYYY-MM-DD HH:MM:SS.fff+TZ :: LEVEL :: PID :: process] message",
      "endOfRun": "RUN SUMMARY with 'Export: n/m tables succeeded' and the failed tables"
    },
    "integration": {
      "scheduling": "Run the same sync command from cron, Windows Task Scheduler, Airflow or another scheduler; exit codes 10-29 need a fix, 40 can be retried with --resume",
      "concurrency": "One run of a configuration at a time; with a SQLite or DuckDB LakeXpress DB, all syncs on one machine, one at a time",
      "ci": "Supply the licence with the LAKEXPRESS_LICENSE environment variable",
      "sbom": "https://lakexpress-docs.arpe.io/latest/reference/sbom"
    },
    "references": {
      "cliReference": "https://lakexpress-docs.arpe.io/latest/cli/",
      "compatibility": "https://lakexpress-docs.arpe.io/latest/reference/compatibility",
      "logging": "https://lakexpress-docs.arpe.io/latest/reference/logging",
      "fullDocumentationForLLMs": "https://lakexpress-docs.arpe.io/llms-full.txt",
      "agentSkillReferences": "https://github.com/arpe-io/arpeio-skills/tree/main/plugins/lakexpress/skills/lakexpress/references"
    }
  },
  "metadata": {
    "lastUpdated": "2026-10-22",
    "productVersion": "0.4.10",
    "purpose": "Technical specifications for AI agents and developers",
    "schemaVersion": "1.0.0"
  }
}