re-include executor/driver cores in spark-config

stackabletech · Oct 31, 2023 · ac7a3b1 · ac7a3b1
1 parent 4813cfa
commit ac7a3b1
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,6 +25,7 @@ All notable changes to this project will be documented in this file.
 ### Fixed
 
 - Dynamic loading of Maven packages ([#281]).
+- Re-instated drive/executor cores setting ([#xxx]).
 
 ### Removed
 

diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
@@ -825,6 +825,7 @@ fn resources_to_driver_props(
         let min_cores = cores_from_quantity(min.0.clone())?;
         let max_cores = cores_from_quantity(max.0.clone())?;
         // will have default value from resources to apply if nothing set specifically
+        props.insert("spark.driver.cores".to_string(), max_cores.clone());
         props.insert(
             "spark.kubernetes.driver.request.cores".to_string(),
             min_cores,
@@ -864,6 +865,7 @@ fn resources_to_executor_props(
         let min_cores = cores_from_quantity(min.0.clone())?;
         let max_cores = cores_from_quantity(max.0.clone())?;
         // will have default value from resources to apply if nothing set specifically
+        props.insert("spark.executor.cores".to_string(), max_cores.clone());
         props.insert(
             "spark.kubernetes.executor.request.cores".to_string(),
             min_cores,
@@ -1044,6 +1046,7 @@ mod tests {
         resources_to_driver_props(true, &driver_config, &mut props).expect("blubb");
 
         let expected: BTreeMap<String, String> = vec![
+            ("spark.driver.cores".to_string(), "1".to_string()),
             ("spark.driver.memory".to_string(), "128m".to_string()),
             (
                 "spark.kubernetes.driver.limit.cores".to_string(),
@@ -1087,6 +1090,7 @@ mod tests {
         resources_to_executor_props(true, &executor_config, &mut props).expect("blubb");
 
         let expected: BTreeMap<String, String> = vec![
+            ("spark.executor.cores".to_string(), "2".to_string()),
             ("spark.executor.memory".to_string(), "128m".to_string()), // 128 and not 512 because memory overhead is subtracted
             (
                 "spark.kubernetes.executor.request.cores".to_string(),

diff --git a/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2
@@ -25,10 +25,12 @@ spec:
     spark.kubernetes.executor.podNamePrefix: "resources-sparkconf"
     spark.kubernetes.driver.request.cores: "1"
     spark.kubernetes.driver.limit.cores: "1"
+    spark.driver.cores: "1"
     spark.driver.memory: "1g"
     spark.driver.memoryOverheadFactor: "0.4"
     spark.kubernetes.executor.request.cores: "1"
     spark.kubernetes.executor.limit.cores: "2"
+    spark.executor.cores: "2"
     spark.executor.memory: "2g"
     spark.executor.memoryOverheadFactor: "0.4"
     spark.executor.instances: "1"