{
 "name": "Change Budget",
 "edition": "2026-11-10",
 "read_date": "2026-11-10",
 "source": "https://sparkaccelerationcompare.com/datasets/",
 "license": "CC BY 4.0",
 "cite": "Spark Acceleration Compare, \"Spark Accelerator Ledger\" and \"Change Budget\", edition 10 Nov 2026, https://sparkaccelerationcompare.com/datasets/ (CC BY 4.0).",
 "description": "What each option makes a team change, in four columns: code, cluster, cloud and vendor. Alphabetical by option.",
 "rows": [
  {
   "option": "Amazon EMR runtime for Apache Spark",
   "column": "code",
   "label": "No code change stated",
   "detail": "AWS describes the runtime as 100% API compatible with open source Spark, so Spark jobs need no code changes.",
   "source_url": "https://aws.amazon.com/emr/features/spark/",
   "read_date": "2026-11-10"
  },
  {
   "option": "Amazon EMR runtime for Apache Spark",
   "column": "cluster",
   "label": "Run the jobs on an EMR release",
   "detail": "Jobs must run on EMR (EC2, EKS or Serverless); a team on other Spark hosts migrates first. AQE-related features are on by default, and some can be tuned with Spark properties.",
   "source_url": "https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-spark-performance.html",
   "read_date": "2026-11-10"
  },
  {
   "option": "Amazon EMR runtime for Apache Spark",
   "column": "cloud",
   "label": "AWS only",
   "detail": "AWS lists the runtime where EMR, EMR on EKS and EMR Serverless are available, and mentions EMR on AWS Outposts.",
   "source_url": "https://aws.amazon.com/emr/pricing/",
   "read_date": "2026-11-10"
  },
  {
   "option": "Amazon EMR runtime for Apache Spark",
   "column": "vendor",
   "label": "EMR charges added to EC2 and EBS",
   "detail": "The EMR price is added to the EC2 and EBS price. EMR on EKS is charged per vCPU-hour and GB-hour requested, plus EKS and EC2 or Fargate cost.",
   "source_url": "https://aws.amazon.com/emr/pricing/",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache Auron",
   "column": "code",
   "label": "No job code change stated",
   "detail": "Auron is added with a jar on the Spark classpath and three settings (spark.auron.enable, the AuronSparkSessionExtension and the AuronShuffleManager); the project states no job changes.",
   "source_url": "https://auron.apache.org/",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache Auron",
   "column": "cluster",
   "label": "Build the jar, set three configs",
   "detail": "You must build the jar yourself (Rust nightly and a JDK). The quick start disables Spark off-heap memory and moves memory into executor overhead (4096 MB).",
   "source_url": "https://auron.apache.org/documents/benchmarks.html",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache Auron",
   "column": "cloud",
   "label": "None found on 2026-11-10",
   "detail": "No cloud requirement is stated. The homepage says it supports storage such as HDFS and S3, and no cloud-specific instance guidance was found. Documented support for EMR, EKS or Databricks: none found on 2026-11-10.",
   "source_url": "https://raw.githubusercontent.com/apache/auron/master/README.md",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache Auron",
   "column": "vendor",
   "label": "Apache-2.0, no contract stated",
   "detail": "Auron is Apache-2.0 licensed under the Apache Software Foundation; no vendor or platform contract is described.",
   "source_url": "https://auron.apache.org/",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache DataFusion Comet",
   "column": "code",
   "label": "No job code change stated",
   "detail": "Comet is enabled with spark.plugins, a shuffle manager and off-heap settings, with no job code changes. Iceberg writes are off by default.",
   "source_url": "https://raw.githubusercontent.com/apache/datafusion-comet/main/docs/source/user-guide/latest/compatibility/index.md",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache DataFusion Comet",
   "column": "cluster",
   "label": "Plugin, shuffle manager, off-heap memory",
   "detail": "Every deployment must configure how much memory Comet can use. Supported versions are Spark 3.5.9, 4.0.4, 4.1.3 and 4.2.0 with JDK 17 or later.",
   "source_url": "https://raw.githubusercontent.com/apache/datafusion-comet/main/docs/source/user-guide/latest/tuning.md",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache DataFusion Comet",
   "column": "cloud",
   "label": "None found on 2026-11-10",
   "detail": "No cloud requirement is stated. The published jars cover Linux on amd64 and arm64; the benchmark ran on EKS with r6i instances.",
   "source_url": "https://raw.githubusercontent.com/apache/datafusion-comet/main/docs/source/user-guide/latest/installation.md",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache DataFusion Comet",
   "column": "vendor",
   "label": "Apache-2.0, no contract stated",
   "detail": "Comet is Apache-2.0 licensed under the Apache Software Foundation; no vendor or platform contract is described.",
   "source_url": "https://api.github.com/repos/apache/datafusion-comet",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache Gluten with Velox",
   "column": "code",
   "label": "No DataFrame or SQL changes",
   "detail": "Gluten needs no DataFrame or SQL changes, only configuration.",
   "source_url": "https://raw.githubusercontent.com/apache/gluten/main/docs/get-started/getting-started.md",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache Gluten with Velox",
   "column": "cluster",
   "label": "Plugin, off-heap memory, shuffle manager",
   "detail": "The settings are spark.plugins=org.apache.gluten.GlutenPlugin, off-heap memory enabled with a size, ColumnarShuffleManager and the bundle jar loaded ahead of Spark's jars.",
   "source_url": "https://raw.githubusercontent.com/apache/gluten/main/docs/get-started/getting-started.md",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache Gluten with Velox",
   "column": "cloud",
   "label": "None found on 2026-11-10",
   "detail": "No cloud requirement is stated. Official build and test platforms are Ubuntu 20.04/22.04 or CentOS 8, and the docs have pages for S3, GCS and ABFS storage.",
   "source_url": "https://gluten.apache.org/downloads/",
   "read_date": "2026-11-10"
  },
  {
   "option": "Apache Gluten with Velox",
   "column": "vendor",
   "label": "Apache-2.0, no contract stated",
   "detail": "Gluten's project license is Apache-2.0; no vendor or platform contract is described.",
   "source_url": "https://gluten.apache.org/poweredby/",
   "read_date": "2026-11-10"
  },
  {
   "option": "Databricks Photon",
   "column": "code",
   "label": "No change, with listed exceptions",
   "detail": "Databricks says Photon works with existing Spark API code with no changes. It does not support UDFs, RDD APIs or Dataset APIs, and stateful streaming is not supported.",
   "source_url": "https://docs.databricks.com/en/compute/photon.html",
   "read_date": "2026-11-10"
  },
  {
   "option": "Databricks Photon",
   "column": "cluster",
   "label": "Set runtime_engine to PHOTON in API",
   "detail": "Photon is on by default for classic all-purpose, jobs compute and classic pipelines. With the Clusters or Jobs API you must set runtime_engine to PHOTON. Photon instance types use a different DBU rate.",
   "source_url": "https://docs.databricks.com/en/compute/photon.html",
   "read_date": "2026-11-10"
  },
  {
   "option": "Databricks Photon",
   "column": "cloud",
   "label": "Databricks on AWS, Azure or GCP",
   "detail": "Photon is documented as part of Databricks compute on AWS, Azure and Google Cloud.",
   "source_url": "https://docs.databricks.com/en/compute/photon.html",
   "read_date": "2026-11-10"
  },
  {
   "option": "Databricks Photon",
   "column": "vendor",
   "label": "Databricks platform required",
   "detail": "Photon runs only inside Databricks. Photon instance types consume DBUs at a different rate than the same instance without Photon, so cost depends on the DBU multiplier on the pricing page.",
   "source_url": "https://www.databricks.com/product/pricing",
   "read_date": "2026-11-10"
  },
  {
   "option": "DataPelago",
   "column": "code",
   "label": "No application rewrite stated",
   "detail": "DataPelago presents DPA-S as working with existing Spark clusters and jobs with no application rewrites or data migration. It tells users to avoid heavy use of RDDs and UDFs.",
   "source_url": "https://www.datapelago.ai/dpa",
   "read_date": "2026-11-10"
  },
  {
   "option": "DataPelago",
   "column": "cluster",
   "label": "Plugin, license and memory settings",
   "detail": "Mandatory settings include the Gluten plugin class, the subscription ID and license location, off-heap memory, the columnar shuffle manager and Netty Java options. On AWS it ships as a Docker container in ECR for Spark 3.5.1 on EKS.",
   "source_url": "https://datapelago-docs.readthedocs-hosted.com/en/latest/deploy-on-aws.html",
   "read_date": "2026-11-10"
  },
  {
   "option": "DataPelago",
   "column": "cloud",
   "label": "AWS EKS or Google Cloud",
   "detail": "DataPelago documents EKS on AWS and Dataproc and GKE on Google Cloud, in the customer's own account.",
   "source_url": "https://datapelago-docs.readthedocs-hosted.com/en/latest/deploy-on-gcp.html",
   "read_date": "2026-11-10"
  },
  {
   "option": "DataPelago",
   "column": "vendor",
   "label": "Marketplace subscription required",
   "detail": "DPA-S is billed pay-as-you-go through the AWS and GCP marketplaces and needs a Subscription ID and license bucket tied to registered AWS account IDs. No unit price is published on the pages read.",
   "source_url": "https://datapelago-docs.readthedocs-hosted.com/en/latest/deploy-on-aws.html",
   "read_date": "2026-11-10"
  },
  {
   "option": "DualBird",
   "column": "code",
   "label": "No code change stated",
   "detail": "DualBird's about page says 'No infrastructure changes. No code changes.'",
   "source_url": "https://dualbird.io/about",
   "read_date": "2026-11-10"
  },
  {
   "option": "DualBird",
   "column": "cluster",
   "label": "Instance type change to F2",
   "detail": "DualBird's onboarding is two changes: switch the EC2 instance type to F2 and add the DualBird Spark plugin. Cluster sizing then reduces to the number of F2 nodes, per DualBird.",
   "source_url": "https://dualbird.io/product",
   "read_date": "2026-11-10"
  },
  {
   "option": "DualBird",
   "column": "cloud",
   "label": "AWS only, F2 in eight regions",
   "detail": "F2 is listed in us-east-1, us-west-2, ap-northeast-1, ap-northeast-2, ap-southeast-2, ca-central-1, eu-central-1 and eu-west-2, and not, for example, in us-east-2 or eu-west-1. DualBird mentions no other cloud.",
   "source_url": "https://docs.aws.amazon.com/ec2/latest/instancetypes/ec2-instance-regions.html",
   "read_date": "2026-11-10"
  },
  {
   "option": "DualBird",
   "column": "vendor",
   "label": "Plugin contract; no price published",
   "detail": "No pricing, price list or marketplace listing was found on DualBird's site, and the pricing model for the plugin is not stated.",
   "source_url": "https://dualbird.io/resources/blog/iceberg-compaction-benchmark-faster-spark-lower-cost",
   "read_date": "2026-11-10"
  },
  {
   "option": "Flarion",
   "column": "code",
   "label": "No code change stated",
   "detail": "Flarion says UDFs and UDAFs run on Spark as before and operations it does not accelerate run on Spark automatically. The home page headline says 'without changing a line of code'.",
   "source_url": "https://www.flarion.io/faqs",
   "read_date": "2026-11-10"
  },
  {
   "option": "Flarion",
   "column": "cluster",
   "label": "One JAR and one config change",
   "detail": "Flarion is one JAR inside the existing cluster, installed with a one config change and switched off with one config change. Existing Spark tuning carries over.",
   "source_url": "https://www.flarion.io/faqs",
   "read_date": "2026-11-10"
  },
  {
   "option": "Flarion",
   "column": "cloud",
   "label": "No single cloud required",
   "detail": "Flarion lists Databricks, Amazon EMR, Google Cloud Dataproc, Azure HDInsight, Spark on Kubernetes and on-premises, and says it supports air-gapped setups.",
   "source_url": "https://www.flarion.io/product",
   "read_date": "2026-11-10"
  },
  {
   "option": "Flarion",
   "column": "vendor",
   "label": "Flarion license at usage-based prices",
   "detail": "Flarion lists $0.009 per CPU hour (self-hosted), $0.012 (fully managed Spark), $0.095 per DBU (Databricks) and $0.75 per CPU per month (on premises), billed on accelerated Spark cores.",
   "source_url": "https://www.flarion.io/pricing",
   "read_date": "2026-11-10"
  },
  {
   "option": "Google Cloud Lightning Engine",
   "column": "code",
   "label": "No Spark code change stated",
   "detail": "Google's documentation says Lightning Engine needs no Spark code changes. Spark, PySpark, SparkSQL and SparkR jobs are accelerated; other job types run on the standard engine.",
   "source_url": "https://docs.cloud.google.com/managed-spark/docs/guides/lightning-engine",
   "read_date": "2026-11-10"
  },
  {
   "option": "Google Cloud Lightning Engine",
   "column": "cluster",
   "label": "Image 2.3.3+ or 3.0.2+, engine flag",
   "detail": "Clusters need image 2.3.3 or later, or 3.0.2 or later (debian13 or ubuntu24), and the flag --engine=lightning at creation. NQE jobs on GPU accelerators fail.",
   "source_url": "https://docs.cloud.google.com/managed-spark/docs/guides/lightning-engine",
   "read_date": "2026-11-10"
  },
  {
   "option": "Google Cloud Lightning Engine",
   "column": "cloud",
   "label": "Google Cloud only",
   "detail": "Google lists Lightning Engine on Managed Service for Apache Spark, with regional rates (us-central1 as the example).",
   "source_url": "https://docs.cloud.google.com/managed-spark/pricing",
   "read_date": "2026-11-10"
  },
  {
   "option": "Google Cloud Lightning Engine",
   "column": "vendor",
   "label": "Premium tier or cluster add-on",
   "detail": "Serverless premium is $0.089 per DCU-hour against $0.06 standard in us-central1. Clusters pay a management fee plus a Lightning Engine add-on plus Compute Engine VMs and disks.",
   "source_url": "https://docs.cloud.google.com/managed-spark/pricing",
   "read_date": "2026-11-10"
  },
  {
   "option": "Microsoft Fabric native execution engine",
   "column": "code",
   "label": "No code modifications stated",
   "detail": "Microsoft's documentation says no code modifications are required. Unsupported operators fall back to JVM Spark.",
   "source_url": "https://learn.microsoft.com/en-us/fabric/data-engineering/native-execution-engine-overview",
   "read_date": "2026-11-10"
  },
  {
   "option": "Microsoft Fabric native execution engine",
   "column": "cluster",
   "label": "Toggle in environment or Spark config",
   "detail": "Enable it with the Acceleration tab toggle in an environment, or set spark.native.enabled to true in a notebook or Spark job definition. It needs Runtime 1.3 or 2.0.",
   "source_url": "https://learn.microsoft.com/en-us/fabric/data-engineering/native-execution-engine-overview",
   "read_date": "2026-11-10"
  },
  {
   "option": "Microsoft Fabric native execution engine",
   "column": "cloud",
   "label": "Microsoft Fabric on Azure only",
   "detail": "The engine runs only inside Microsoft Fabric on OneLake data, and does not run on AWS EMR or EKS.",
   "source_url": "https://learn.microsoft.com/en-us/fabric/data-engineering/native-execution-engine-overview",
   "read_date": "2026-11-10"
  },
  {
   "option": "Microsoft Fabric native execution engine",
   "column": "vendor",
   "label": "Fabric capacity required",
   "detail": "Fabric is billed by capacity units (F2 to F4096 SKUs), pay-as-you-go or reservation. Autoscale Billing for Spark still requires a base Fabric capacity. SKU prices were not readable on the page.",
   "source_url": "https://azure.microsoft.com/en-us/pricing/details/microsoft-fabric/",
   "read_date": "2026-11-10"
  },
  {
   "option": "NVIDIA cuDF plugin for Apache Spark",
   "column": "code",
   "label": "No code change stated",
   "detail": "NVIDIA says existing Spark applications run with no code change once the plugin is launched and enabled by a configuration setting. Unsupported operators and UDFs fall back to the CPU.",
   "source_url": "https://docs.nvidia.com/spark-rapids/user-guide/latest/overview.html",
   "read_date": "2026-11-10"
  },
  {
   "option": "NVIDIA cuDF plugin for Apache Spark",
   "column": "cluster",
   "label": "GPU instances and plugin settings",
   "detail": "The cluster needs NVIDIA GPUs (Volta to Blackwell generations) and a current driver. AWS's test used g7.4xlarge nodes with executor memoryOverhead 30G, pinnedPool 8G, concurrentGpuTasks 3 and RapidsShuffleManager.",
   "source_url": "https://aws.amazon.com/blogs/big-data/gpu-accelerated-apache-spark-with-amazon-emr-and-nvidia-rtx-pro-4500-on-amazon-ec2-g7-instances-runs-up-to-3-7x-faster/",
   "read_date": "2026-11-10"
  },
  {
   "option": "NVIDIA cuDF plugin for Apache Spark",
   "column": "cloud",
   "label": "AWS, Azure and Google Cloud documented",
   "detail": "NVIDIA lists Databricks runtimes on AWS and Azure and Dataproc on Google Cloud; Amazon EMR on EKS supports the plugin natively from release 6.9.0.",
   "source_url": "https://nvidia.github.io/cudf-spark/docs/download.html",
   "read_date": "2026-11-10"
  },
  {
   "option": "NVIDIA cuDF plugin for Apache Spark",
   "column": "vendor",
   "label": "Free software; GPU instances paid",
   "detail": "The software is free (source Apache-2.0); you pay for GPU instances, and AWS's cost per run includes EC2 and EMR on EKS charges.",
   "source_url": "https://aws.amazon.com/blogs/big-data/gpu-accelerated-apache-spark-with-amazon-emr-and-nvidia-rtx-pro-4500-on-amazon-ec2-g7-instances-runs-up-to-3-7x-faster/",
   "read_date": "2026-11-10"
  },
  {
   "option": "Speedata",
   "column": "code",
   "label": "No code change stated",
   "detail": "Speedata says zero code changes. The Dash plugin identifies compute-intensive work and offloads it, delegating UDFs to the CPU.",
   "source_url": "https://www.speedata.io/product",
   "read_date": "2026-11-10"
  },
  {
   "option": "Speedata",
   "column": "cluster",
   "label": "PCIe C200 card in each server",
   "detail": "The APU is a PCIe Gen 5 card (C200 with the Callisto ASIC), so it needs physical servers. Enabling it uses a jar and two settings: spark.plugins=com.speedata.spark.DashPlugin and spark.speedata.apus=1.",
   "source_url": "https://www.speedata.io/product",
   "read_date": "2026-11-10"
  },
  {
   "option": "Speedata",
   "column": "cloud",
   "label": "On-premises, OEM or hosted AWS",
   "detail": "Options are a 2U server with two C200 cards, OEM servers from Dell and HPE, or Speedata Acceleration for AWS, which runs the work in a Speedata-hosted environment and syncs data from S3.",
   "source_url": "https://www.speedata.io/",
   "read_date": "2026-11-10"
  },
  {
   "option": "Speedata",
   "column": "vendor",
   "label": "Speedata purchase; no price list",
   "detail": "No price list is published. The pages offer buying a server or going through OEM or cloud partners.",
   "source_url": "https://www.speedata.io/product",
   "read_date": "2026-11-10"
  }
 ]
}