{ "$schema": "https://json-schema.org/draft/2020-12/schema", "useCases": { "conventions": { "note": "Commands use Linux syntax and LakeXpress 0.4.10. Entry names such as lxdb_postgres, source_postgres, s3_01 or snowflake_prod are top-level keys of credentials.json that you choose. is the id printed by config create, or the one set with --sync_id. The trial edition finds its bundled FastBCP; with the full edition add --fastbcp_dir_path to config create.", "windows": "Use .\\LakeXpress.exe and the PowerShell backtick ` for line continuation" }, "primary": [ { "id": "first-local-export", "title": "First export: a PostgreSQL schema to local Parquet", "category": "Getting started", "problem": "You want to try LakeXpress on one schema without any cloud account.", "solution": "Create the LakeXpress DB tables, a sync configuration writing to a local directory, then run it.", "commands": [ "./LakeXpress lxdb init --auth_file credentials.json --lxdb_auth_id lxdb_postgres", "./LakeXpress config create --auth_file credentials.json --lxdb_auth_id lxdb_postgres --source_db_auth_id source_postgres --source_schema_name public --output_dir ./exports --n_jobs 4 --fastbcp_p 2", "./LakeXpress sync --auth_file credentials.json --lxdb_auth_id lxdb_postgres --sync_id ", "./LakeXpress status --auth_file credentials.json --lxdb_auth_id lxdb_postgres --sync_id " ], "result": "Each table is written under exports/public// as Parquet. 4 tables run in parallel, large tables split in 2.", "tip": "Without a database server, use a SQLite LakeXpress DB entry: {\"ds_type\": \"sqlite\", \"auth_mode\": \"filesystem\", \"info\": {\"filepath\": \"/abs/path/lx.sqlite\"}}", "docs": "https://lakexpress-docs.arpe.io/latest/guides/quickstart" }, { "id": "bronze-layer-cloud-storage", "title": "Feed the bronze layer of a data lake", "category": "Data lake", "problem": "Source schemas must land as Parquet in S3, ADLS Gen2, GCS or OneLake, in a predictable folder layout.", "solution": "Point the configuration at a cloud storage entry, filter schemas and tables, and add a sub-path.", "commands": [ "./LakeXpress config create --auth_file credentials.json --lxdb_auth_id lxdb_postgres --source_db_auth_id source_postgres --source_schema_name \"sales%, marketing%\" --include \"fact_%, dim_%\" --exclude \"temp%, test%\" --target_storage_id s3_01 --sub_path data-lake/prod --n_jobs 8 --fastbcp_p 4" ], "result": "Files under s3:////data-lake/prod//
/", "docs": "https://lakexpress-docs.arpe.io/latest/configuration/storage/storage-overview" }, { "id": "publish-snowflake", "title": "Publish a schema to Snowflake", "category": "Publishing", "problem": "Tables must be queryable in Snowflake, either over the files or loaded.", "solution": "Stage the files in storage behind a Snowflake stage, and add a Snowflake publish target. Default is external tables with typed V_
views; --publish_method internal loads them with COPY INTO.", "commands": [ "./LakeXpress config create --auth_file credentials.json --lxdb_auth_id lxdb_postgres --source_db_auth_id source_postgres --source_schema_name tpch_1 --target_storage_id s3_01 --publish_target snowflake_prod --n_jobs 4", "./LakeXpress sync --auth_file credentials.json --lxdb_auth_id lxdb_postgres --sync_id " ], "result": "Schema EXT_tpch_1 with external tables and V_
views", "requirement": "The storage entry's directory must equal the Snowflake stage URL", "docs": "https://lakexpress-docs.arpe.io/latest/integrations/snowflake-publishing" }, { "id": "publish-databricks", "title": "Load SQL Server tables into Databricks as managed Delta tables", "category": "Publishing", "problem": "Large SQL Server tables must reach Databricks without running a Spark cluster for the copy.", "solution": "Export to S3 and publish as internal tables in the Unity Catalog catalog of the Databricks entry.", "commands": [ "./LakeXpress config create --auth_file credentials.json --lxdb_auth_id lxdb_postgres --source_db_auth_id source_mssql --source_schema_name dbo --target_storage_id s3_01 --publish_target databricks_prod --publish_method internal --publish_schema_pattern \"bronze_{schema}\" --n_jobs 4" ], "result": "Tables .bronze_dbo.
", "docs": "https://lakexpress-docs.arpe.io/latest/integrations/databricks-publishing" }, { "id": "publish-fabric", "title": "Publish to a Microsoft Fabric Lakehouse", "category": "Publishing", "problem": "Source tables must appear as Delta tables in a Fabric Lakehouse for the SQL endpoint and Power BI.", "solution": "Write to OneLake and publish internal tables. The Lakehouse namespace is flat, so prefix tables with their schema.", "commands": [ "./LakeXpress config create --auth_file credentials.json --lxdb_auth_id lxdb_postgres --source_db_auth_id source_postgres --source_schema_name tpch_1 --target_storage_id onelake_01 --publish_target fabric_lakehouse --publish_method internal --publish_table_pattern \"{schema}_{table}\" --n_jobs 4 --fastbcp_p 2" ], "requirement": "The fabric_lakehouse entry has \"fabric_target\": \"lakehouse\"", "docs": "https://lakexpress-docs.arpe.io/latest/integrations/fabric-publishing" }, { "id": "publish-glue", "title": "Oracle to S3, registered in AWS Glue for Athena", "category": "Publishing", "problem": "Oracle tables must be queryable from Athena, Redshift Spectrum or EMR.", "solution": "Export to S3 and register external tables in a Glue database. Split the biggest table with Rowid.", "commands": [ "./LakeXpress config create --auth_file credentials.json --lxdb_auth_id lxdb_postgres --source_db_auth_id source_oracle --source_schema_name SALES --target_storage_id s3_01 --publish_target glue_catalog --publish_database_name analytics --publish_table_pattern \"{schema}_{table}\" --fastbcp_table_config \"SALES.ORDERS:Rowid::8\" --n_jobs 4 --fastbcp_p 4" ], "docs": "https://lakexpress-docs.arpe.io/latest/integrations/glue-publishing" }, { "id": "publish-bigquery", "title": "PostgreSQL to BigQuery native tables", "category": "Publishing", "problem": "Tables must be loaded into a BigQuery dataset.", "solution": "Export to GCS and publish internal tables (load jobs).", "commands": [ "./LakeXpress config create --auth_file credentials.json --lxdb_auth_id lxdb_postgres --source_db_auth_id source_postgres --source_schema_name tpch_1 --target_storage_id gcs_01 --publish_target bigquery_prod --publish_method internal --publish_schema_pattern \"lx_{schema}\" --n_jobs 4" ], "result": "Dataset lx_tpch_1", "docs": "https://lakexpress-docs.arpe.io/latest/integrations/bigquery-publishing" }, { "id": "incremental-upsert", "title": "Scheduled incremental sync with upsert", "category": "Incremental", "problem": "Large tables change every day; re-exporting them in full each time is too slow.", "solution": "Declare watermark columns per table. The first run is a full baseline, later runs export only new rows; upsert merges them on the primary key.", "commands": [ "./LakeXpress config create --auth_file credentials.json --lxdb_auth_id lxdb_postgres --source_db_auth_id source_postgres --source_schema_name sales --target_storage_id s3_01 --incremental_table \"sales.orders:updated_at:datetime!upsert\" --incremental_table \"sales.order_items:created_at:datetime\" --incremental_safety_lag 600 --publish_target snowflake_prod --publish_method internal --sync_id sales-incremental --n_jobs 4", "./LakeXpress sync --auth_file credentials.json --lxdb_auth_id lxdb_postgres --sync_id sales-incremental" ], "notes": [ "sales.orders needs a primary key for upsert", "Other tables of the sales schema are fully exported on each run", "upsert and delete_insert are not supported on AWS Glue or Fabric Lakehouse", "Use a PostgreSQL, SQL Server or SQLite LakeXpress DB" ], "docs": "https://lakexpress-docs.arpe.io/latest/advanced/incremental-sync" }, { "id": "time-partitioned-export", "title": "Large historical tables split by month", "category": "Performance", "problem": "Very large fact tables should be exported in parallel and land in Hive-style folders.", "solution": "Use the Timepartition method per table.", "commands": [ "./LakeXpress config create --auth_file credentials.json --lxdb_auth_id lxdb_postgres --source_db_auth_id source_teradata --source_schema_name claims --output_dir /data/claims --fastbcp_table_config \"CARRIER_CLAIMS:Timepartition:(CLM_FROM_DT,year,month):4\" --n_jobs 4" ], "result": "Files under .../year=YYYY/month=MM/", "docs": "https://lakexpress-docs.arpe.io/latest/advanced/export-performance" }, { "id": "resume-after-failure", "title": "Resume after a partial failure", "category": "Operations", "problem": "A sync ended with exit code 40: some tables failed.", "solution": "Fix the cause, then resume: tables already exported are skipped, and the run publishes once complete.", "commands": [ "./LakeXpress sync --auth_file credentials.json --lxdb_auth_id lxdb_postgres --sync_id --resume", "./LakeXpress cleanup --auth_file credentials.json --lxdb_auth_id lxdb_postgres --sync_id --status running --dry-run" ], "notes": [ "cleanup removes runs left in 'running' after a crash, which otherwise block the next runs", "After a failed run, a plain sync publishes the latest completed run: resume first" ], "docs": "https://lakexpress-docs.arpe.io/latest/guides/recipes/resume-failed-exports" } ], "summary": { "totalPrimaryUseCases": 10, "commonThemes": [ "Schema-level export to Parquet", "Cloud lakehouse ingestion", "Incremental recurring syncs", "Run tracking and resume" ], "moreExamples": "https://lakexpress-docs.arpe.io/latest/guides/command-cookbook" } }, "metadata": { "lastUpdated": "2026-10-22", "productVersion": "0.4.10", "purpose": "Use case library for AI agents", "schemaVersion": "1.0.0" } }