{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "features": {
    "items": [
      {
        "id": "sync-configurations",
        "name": "Sync configurations",
        "category": "Simplicity",
        "description": "A pipeline is defined once with config create and stored in the LakeXpress DB under a sync_id. It is then run with sync, with the same command every time.",
        "benefits": [
          "One command per run, easy to schedule",
          "Configurations can be listed (config list) and kept per environment (--env_name)",
          "Each run writes a YAML copy of its configuration next to its log file"
        ],
        "technicalDetails": {
          "create": "LakeXpress config create --auth_file ... --lxdb_auth_id ... --source_db_auth_id ... (--output_dir or --target_storage_id)",
          "run": "LakeXpress sync --auth_file ... --lxdb_auth_id ... --sync_id ...",
          "syncIdFormat": "1 to 64 letters, digits, _ or -; generated if omitted",
          "editing": "A configuration cannot be edited: delete it (config delete --confirm) and create it again; incremental watermarks then restart"
        }
      },
      {
        "id": "parallel-export",
        "name": "Parallel export",
        "category": "Performance",
        "description": "Tables are exported several at a time, and large tables are split across several FastBCP processes. The export method is chosen per table automatically and can be overridden.",
        "benefits": [
          "Whole schemas exported in one run",
          "Large tables split without manual partitioning",
          "Streaming export: tables are not held in memory"
        ],
        "technicalDetails": {
          "tablesInParallel": "--n_jobs (default 1)",
          "processesPerTable": "--fastbcp_p (default 1), applied to tables with at least --large_table_threshold rows (default 100000)",
          "methodsWithoutKey": ["None", "Ctid (PostgreSQL)", "Rowid (Oracle)", "Physloc (SQL Server)"],
          "methodsWithKey": ["Random", "DataDriven", "RangeId", "Ntile", "Timepartition"],
          "perTableOverride": "--fastbcp_table_config \"[schema.]table:method:key_column:degree\" or rows in the partition_columns table of the LakeXpress DB",
          "priority": "partition_columns > --fastbcp_table_config > automatic"
        }
      },
      {
        "id": "incremental-sync",
        "name": "Incremental sync",
        "category": "Efficiency",
        "description": "LakeXpress stores a high watermark per table and, on later syncs, exports only the rows above it. Tables not listed as incremental are fully exported on each sync.",
        "benefits": [
          "Short recurring runs on large, growing tables",
          "Less load on the source and the network",
          "Updates applied in the target with upsert or delete_insert"
        ],
        "technicalDetails": {
          "spec": "--incremental_table \"schema.table:column:type[:i|:e][@start][!strategy]\" (repeat per table)",
          "columnTypes": ["date", "datetime", "int", "bigint", "auto"],
          "bounds": ":e exclusive (column > watermark, default) or :i inclusive (column >= watermark)",
          "loadingStrategies": ["append (default)", "upsert", "delete_insert"],
          "safetyLag": "--incremental_safety_lag SECONDS to stay behind the current time for late rows",
          "locks": "Incremental runs lock their tables in the LakeXpress DB; lxdb locks and lxdb release-locks manage locks left by a crash",
          "lakeXpressDBNote": "Use a PostgreSQL, SQL Server or SQLite LakeXpress DB for incremental syncs"
        }
      },
      {
        "id": "publishing",
        "name": "Publishing to data platforms",
        "category": "Integration",
        "description": "After the export, LakeXpress can create tables in a target platform: external tables that read the Parquet files in storage, or internal tables loaded into the platform.",
        "benefits": [
          "Export and table creation in one run",
          "Same configuration model for eight platforms",
          "Naming patterns for target schemas and tables"
        ],
        "technicalDetails": {
          "targets": ["Snowflake", "Databricks", "Microsoft Fabric", "BigQuery", "Amazon Redshift", "MotherDuck", "AWS Glue", "DuckLake"],
          "method": "--publish_method external (default) or internal; AWS Glue and DuckLake are external only",
          "schemaPattern": "--publish_schema_pattern, tokens {schema} {database} {date} {timestamp} {uuid} {subpath}; default EXT_{schema} (external) or {schema} (internal)",
          "tablePattern": "--publish_table_pattern, must contain {table}; default {table}",
          "primaryKeys": "--pk_constraints adds source primary keys to internal tables",
          "publishLater": "'sync[export]' exports only; 'sync[publish]' publishes the files of a previous run"
        }
      },
      {
        "id": "resume",
        "name": "Resume and run management",
        "category": "Reliability",
        "description": "Every run and every table is recorded in the LakeXpress DB. A run with failed tables can be resumed: tables already exported are skipped.",
        "benefits": [
          "No full re-export after a partial failure",
          "Status of every run available from the command line",
          "Stuck runs can be cleaned up"
        ],
        "technicalDetails": {
          "resume": "sync --sync_id ... --resume [--run_id ...]",
          "status": "status --sync_id ... [--verbose] or status --run_id ...",
          "cleanup": "cleanup --sync_id ... [--older-than 7d] [--status running|failed] [--dry-run]",
          "errorAction": "--error_action fail (default) stops at the first failed table; continue exports the others"
        }
      },
      {
        "id": "filtering",
        "name": "Schema and table filtering",
        "category": "Simplicity",
        "description": "Select what to export with SQL LIKE patterns on schemas and tables, and with row-count bounds.",
        "benefits": [
          "Export a full database or a subset of tables with the same command",
          "Skip temporary or test tables"
        ],
        "technicalDetails": {
          "schemas": "--source_schema_name, comma-separated, LIKE patterns allowed; default all schemas",
          "include": "--include \"orders%, customer%\"",
          "exclude": "--exclude \"temp%, test%\"",
          "rowCount": "--min_rows N, --max_rows N"
        }
      },
      {
        "id": "storage-layout",
        "name": "Parquet output and storage layout",
        "category": "Integration",
        "description": "Each table is written as Parquet files under <base>/[<sub_path>/]<schema>/<table>/, on local disk or cloud storage.",
        "benefits": [
          "Predictable folder layout for downstream tools",
          "Extract once, publish to several targets or re-publish without re-extracting"
        ],
        "technicalDetails": {
          "local": "--output_dir PATH (saved as an absolute path from 0.4.9)",
          "cloud": "--target_storage_id ID (S3, GCS, Azure, OneLake entry of the credentials file)",
          "subPath": "--sub_path adds folders between the base path and the schema folders",
          "compression": "--compression_type Zstd (default), Snappy, Gzip, Lz4, None",
          "hivePartitions": "The Timepartition method writes year=YYYY/month=MM/... folders"
        }
      },
      {
        "id": "cdm-metadata",
        "name": "CDM metadata",
        "category": "Integration",
        "description": "Optionally writes Common Data Model files (manifest.json and <table>.cdm.json) next to the Parquet files.",
        "benefits": [
          "Schema description travels with the data"
        ],
        "technicalDetails": {
          "flag": "--generate_metadata",
          "manifestName": "--manifest_name (default: schema or database name)"
        }
      },
      {
        "id": "logging",
        "name": "Logging and run tracking",
        "category": "Operations",
        "description": "Console and log file output for each run, and structured tracking tables in the LakeXpress DB (runs, jobs per table, files, columns, events).",
        "benefits": [
          "Audit trail of what was exported, when, and how many rows",
          "SQL access to run history for monitoring"
        ],
        "technicalDetails": {
          "logFile": "lx_YYYYMMDD_HHMMSS_<run_id>.log in --log_dir (default: current directory)",
          "logLevels": ["DEBUG", "INFO (default)", "WARNING", "ERROR"],
          "trackingTables": ["export_runs", "export_jobs", "export_files", "export_columns", "export_log", "export_steps"],
          "exitCodes": "0 success, 40 some tables failed, 10 configuration error, 12 connection error, 13 licence (see technical-specs.json)"
        }
      },
      {
        "id": "credentials-file",
        "name": "Single credentials file with environment variables",
        "category": "Security",
        "description": "All connections (LakeXpress DB, sources, storage, publishing targets) are entries of one JSON file. Any string value can be read from an environment variable with $env{VAR}.",
        "benefits": [
          "Secrets kept out of command lines and configuration",
          "Read-only source user: LakeXpress never writes to the source"
        ],
        "technicalDetails": {
          "flag": "--auth_file (-a) PATH",
          "entryShape": "{ \"ds_type\": ..., \"auth_mode\": ..., \"info\": { ... } }",
          "envSubstitution": "$env{VAR_NAME}; an unset variable raises an error"
        }
      }
    ],
    "websiteHighlights": {
      "source": "https://lakexpress.arpe.io (Built for Enterprise Performance section)",
      "items": [
        { "title": "Simple", "text": "Just one command line to publish a full database or a subset of tables." },
        { "title": "Fast", "text": "Parallel copy technology, even for massive tables." },
        { "title": "Actionable", "text": "Embed anywhere: Linux or Windows VM, Docker containers, Lambda functions, or scheduled jobs." },
        { "title": "Professional", "text": "Enterprise-grade logging to console, JSON files, or databases. Full audit trail for compliance." }
      ]
    },
    "summary": {
      "totalFeatures": 10,
      "categories": ["Simplicity", "Performance", "Efficiency", "Integration", "Reliability", "Operations", "Security"]
    }
  },
  "metadata": {
    "lastUpdated": "2026-10-22",
    "productVersion": "0.4.10",
    "purpose": "Feature catalog for AI agents",
    "schemaVersion": "1.0.0"
  }
}