Skip to content

Commit d8948a6

Browse files
andygroveandygrove
committed
fix: link accumulator-related IgnoreCometNativeScan tests to #3867
Update all tests ignored because native scans cannot propagate JVM-side Spark accumulators to reference the new tracking issue across all three Spark version diffs (3.4.3, 3.5.8, 4.0.1). Affected tests: - filter pushdown - StringPredicate - Filters should be pushed down for vectorized Parquet reader at row group level - SPARK-34562: Bloom filter push down
1 parent 028e5f9 commit d8948a6

3 files changed

Lines changed: 28 additions & 26 deletions

File tree

‎dev/diffs/3.4.3.diff‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2003,7 +2003,7 @@ index 07e2849ce6f..3e73645b638 100644
20032003
ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
20042004
)
20052005
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
2006-
index 104b4e416cd..0897e4f3776 100644
2006+
index 104b4e416cd..f892aa090d6 100644
20072007
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
20082008
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
20092009
@@ -38,6 +38,7 @@ import org.apache.parquet.schema.MessageType
@@ -2033,7 +2033,7 @@ index 104b4e416cd..0897e4f3776 100644
20332033

20342034
- test("Filters should be pushed down for vectorized Parquet reader at row group level") {
20352035
+ test("Filters should be pushed down for vectorized Parquet reader at row group level",
2036-
+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
2036+
+ IgnoreCometNativeScan("https://github.com/apache/datafusion-comet/issues/3867")) {
20372037
import testImplicits._
20382038

20392039
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
@@ -2069,7 +2069,7 @@ index 104b4e416cd..0897e4f3776 100644
20692069

20702070
- test("filter pushdown - StringPredicate") {
20712071
+ test("filter pushdown - StringPredicate",
2072-
+ IgnoreCometNativeScan("cannot be pushed down")) {
2072+
+ IgnoreCometNativeScan("https://github.com/apache/datafusion-comet/issues/3867")) {
20732073
import testImplicits._
20742074
// keep() should take effect on StartsWith/EndsWith/Contains
20752075
Seq(
@@ -2109,7 +2109,7 @@ index 104b4e416cd..0897e4f3776 100644
21092109

21102110
- test("SPARK-34562: Bloom filter push down") {
21112111
+ test("SPARK-34562: Bloom filter push down",
2112-
+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
2112+
+ IgnoreCometNativeScan("https://github.com/apache/datafusion-comet/issues/3867")) {
21132113
withTempPath { dir =>
21142114
val path = dir.getCanonicalPath
21152115
spark.range(100).selectExpr("id * 2 AS id")

‎dev/diffs/3.5.8.diff‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2032,7 +2032,7 @@ index 07e2849ce6f..3e73645b638 100644
20322032
ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
20332033
)
20342034
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
2035-
index 8e88049f51e..b713ccddfcb 100644
2035+
index 8e88049f51e..a9d09a178c1 100644
20362036
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
20372037
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
20382038
@@ -1095,7 +1095,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -2054,7 +2054,7 @@ index 8e88049f51e..b713ccddfcb 100644
20542054

20552055
- test("Filters should be pushed down for vectorized Parquet reader at row group level") {
20562056
+ test("Filters should be pushed down for vectorized Parquet reader at row group level",
2057-
+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
2057+
+ IgnoreCometNativeScan("https://github.com/apache/datafusion-comet/issues/3867")) {
20582058
import testImplicits._
20592059

20602060
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
@@ -2102,16 +2102,17 @@ index 8e88049f51e..b713ccddfcb 100644
21022102
}
21032103
}
21042104
}
2105-
@@ -1699,7 +1714,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2105+
@@ -1699,7 +1714,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
21062106
(attr, value) => sources.StringContains(attr, value))
21072107
}
21082108

21092109
- test("filter pushdown - StringPredicate") {
2110-
+ test("filter pushdown - StringPredicate", IgnoreCometNativeScan("cannot be pushed down")) {
2110+
+ test("filter pushdown - StringPredicate",
2111+
+ IgnoreCometNativeScan("https://github.com/apache/datafusion-comet/issues/3867")) {
21112112
import testImplicits._
21122113
// keep() should take effect on StartsWith/EndsWith/Contains
21132114
Seq(
2114-
@@ -1743,7 +1758,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2115+
@@ -1743,7 +1759,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
21152116
}
21162117
}
21172118

@@ -2121,7 +2122,7 @@ index 8e88049f51e..b713ccddfcb 100644
21212122
val schema = StructType(Seq(
21222123
StructField("a", IntegerType, nullable = false)
21232124
))
2124-
@@ -1952,8 +1968,17 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2125+
@@ -1952,8 +1969,17 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
21252126
val e = intercept[SparkException] {
21262127
sql(s"select a from $tableName where b > 0").collect()
21272128
}
@@ -2141,7 +2142,7 @@ index 8e88049f51e..b713ccddfcb 100644
21412142
}
21422143

21432144
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
2144-
@@ -1984,7 +2009,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2145+
@@ -1984,7 +2010,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
21452146
}
21462147
}
21472148

@@ -2151,17 +2152,17 @@ index 8e88049f51e..b713ccddfcb 100644
21512152
// block 1:
21522153
// null count min max
21532154
// page-0 0 0 99
2154-
@@ -2044,7 +2070,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2155+
@@ -2044,7 +2071,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
21552156
}
21562157
}
21572158

21582159
- test("SPARK-34562: Bloom filter push down") {
21592160
+ test("SPARK-34562: Bloom filter push down",
2160-
+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
2161+
+ IgnoreCometNativeScan("https://github.com/apache/datafusion-comet/issues/3867")) {
21612162
withTempPath { dir =>
21622163
val path = dir.getCanonicalPath
21632164
spark.range(100).selectExpr("id * 2 AS id")
2164-
@@ -2276,7 +2303,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
2165+
@@ -2276,7 +2304,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
21652166
assert(pushedParquetFilters.exists(_.getClass === filterClass),
21662167
s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
21672168

@@ -2174,7 +2175,7 @@ index 8e88049f51e..b713ccddfcb 100644
21742175
} else {
21752176
assert(selectedFilters.isEmpty, "There is filter pushed down")
21762177
}
2177-
@@ -2336,7 +2367,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
2178+
@@ -2336,7 +2368,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
21782179
assert(pushedParquetFilters.exists(_.getClass === filterClass),
21792180
s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
21802181

‎dev/diffs/4.0.1.diff‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2727,7 +2727,7 @@ index cd6f41b4ef4..4b6a17344bc 100644
27272727
ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
27282728
)
27292729
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
2730-
index 6080a5e8e4b..cef477c8b4d 100644
2730+
index 6080a5e8e4b..d589a0b8ee0 100644
27312731
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
27322732
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
27332733
@@ -38,6 +38,7 @@ import org.apache.parquet.schema.MessageType
@@ -2757,7 +2757,7 @@ index 6080a5e8e4b..cef477c8b4d 100644
27572757

27582758
- test("Filters should be pushed down for vectorized Parquet reader at row group level") {
27592759
+ test("Filters should be pushed down for vectorized Parquet reader at row group level",
2760-
+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
2760+
+ IgnoreCometNativeScan("https://github.com/apache/datafusion-comet/issues/3867")) {
27612761
import testImplicits._
27622762

27632763
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
@@ -2787,16 +2787,17 @@ index 6080a5e8e4b..cef477c8b4d 100644
27872787
}
27882788
}
27892789
}
2790-
@@ -1706,7 +1720,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2790+
@@ -1706,7 +1720,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
27912791
(attr, value) => sources.StringContains(attr, value))
27922792
}
27932793

27942794
- test("filter pushdown - StringPredicate") {
2795-
+ test("filter pushdown - StringPredicate", IgnoreCometNativeScan("cannot be pushed down")) {
2795+
+ test("filter pushdown - StringPredicate",
2796+
+ IgnoreCometNativeScan("https://github.com/apache/datafusion-comet/issues/3867")) {
27962797
import testImplicits._
27972798
// keep() should take effect on StartsWith/EndsWith/Contains
27982799
Seq(
2799-
@@ -1750,7 +1764,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2800+
@@ -1750,7 +1765,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
28002801
}
28012802
}
28022803

@@ -2806,7 +2807,7 @@ index 6080a5e8e4b..cef477c8b4d 100644
28062807
val schema = StructType(Seq(
28072808
StructField("a", IntegerType, nullable = false)
28082809
))
2809-
@@ -1940,7 +1955,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2810+
@@ -1940,7 +1956,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
28102811
}
28112812
}
28122813

@@ -2816,7 +2817,7 @@ index 6080a5e8e4b..cef477c8b4d 100644
28162817
withTempPath { dir =>
28172818
val count = 10
28182819
val tableName = "spark_25207"
2819-
@@ -1993,7 +2009,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2820+
@@ -1993,7 +2010,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
28202821
}
28212822
}
28222823

@@ -2826,17 +2827,17 @@ index 6080a5e8e4b..cef477c8b4d 100644
28262827
// block 1:
28272828
// null count min max
28282829
// page-0 0 0 99
2829-
@@ -2053,7 +2070,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2830+
@@ -2053,7 +2071,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
28302831
}
28312832
}
28322833

28332834
- test("SPARK-34562: Bloom filter push down") {
28342835
+ test("SPARK-34562: Bloom filter push down",
2835-
+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
2836+
+ IgnoreCometNativeScan("https://github.com/apache/datafusion-comet/issues/3867")) {
28362837
withTempPath { dir =>
28372838
val path = dir.getCanonicalPath
28382839
spark.range(100).selectExpr("id * 2 AS id")
2839-
@@ -2305,7 +2323,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
2840+
@@ -2305,7 +2324,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
28402841
assert(pushedParquetFilters.exists(_.getClass === filterClass),
28412842
s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
28422843

@@ -2849,7 +2850,7 @@ index 6080a5e8e4b..cef477c8b4d 100644
28492850
} else {
28502851
assert(selectedFilters.isEmpty, "There is filter pushed down")
28512852
}
2852-
@@ -2368,7 +2390,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
2853+
@@ -2368,7 +2391,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
28532854
assert(pushedParquetFilters.exists(_.getClass === filterClass),
28542855
s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
28552856

0 commit comments

Comments
 (0)