[jira] [Created] (SPARK-21557) Debug issues for SparkML(scala.Predef$.any2ArrowAssoc)

["prabir bhowmick (JIRA)" <ji...@apache.org>]

prabir bhowmick created SPARK-21557:
---------------------------------------

             Summary: Debug issues for SparkML(scala.Predef$.any2ArrowAssoc)
                 Key: SPARK-21557
                 URL: https://issues.apache.org/jira/browse/SPARK-21557
             Project: Spark
          Issue Type: Bug
          Components: MLlib
    Affects Versions: 2.1.1
            Reporter: prabir bhowmick
            Priority: Critical
             Fix For: 2.1.2


Hi Team,

Can you please see the below error ,when I am running the below program using below mvn config.Kindly tell me which version I have to use.I am running this program from eclipse neon.

Error at Runtime:- 

Exception in thread "main" java.lang.NoSuchMethodError: scala.Predef$.any2ArrowAssoc(Ljava/lang/Object;)Ljava/lang/Object;
	at org.apache.spark.sql.SparkSession$Builder.config(SparkSession.scala:750)
	at org.apache.spark.sql.SparkSession$Builder.appName(SparkSession.scala:741)
	at com.MLTest.JavaPCAExample.main(JavaPCAExample.java:20)

Java Class:-

package com.MLTest;

import org.apache.spark.sql.SparkSession;

import java.util.Arrays;
import java.util.List;
import org.apache.spark.ml.feature.PCA;
import org.apache.spark.ml.feature.PCAModel;
import org.apache.spark.ml.linalg.VectorUDT;
import org.apache.spark.ml.linalg.Vectors;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;

public class JavaPCAExample {
	public static void main(String[] args) {
		SparkSession spark = SparkSession.builder().appName("JavaPCAExample3")
				.config("spark.some.config.option", "some-value").getOrCreate();

		List<Row> data = Arrays.asList(
				RowFactory.create(Vectors.sparse(5, new int[] { 1, 3 }, new double[] { 1.0, 7.0 })),
				RowFactory.create(Vectors.dense(2.0, 0.0, 3.0, 4.0, 5.0)),
				RowFactory.create(Vectors.dense(4.0, 0.0, 0.0, 6.0, 7.0)));

		StructType schema = new StructType(
				new StructField[] { new StructField("features", new VectorUDT(), false, Metadata.empty()), });

		Dataset<Row> df = spark.createDataFrame(data, schema);

		PCAModel pca = new PCA().setInputCol("features").setOutputCol("pcaFeatures").setK(3).fit(df);

		Dataset<Row> result = pca.transform(df).select("pcaFeatures");
		result.show(false);

		spark.stop();
	}
}

pom.xml:-

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>SparkMLTest</groupId>
	<artifactId>SparkMLTest</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<build>
		<sourceDirectory>src</sourceDirectory>
		<plugins>
			<plugin>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>3.5.1</version>
				<configuration>
					<source>1.8</source>
					<target>1.8</target>
				</configuration>
			</plugin>
		</plugins>
	</build>
	<dependencies>
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-core_2.10</artifactId>
			<version>2.2.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-streaming_2.10</artifactId>
			<version>2.1.1</version>
		</dependency>
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-mllib_2.10</artifactId>
			<version>2.1.1</version>
			<scope>provided</scope>
		</dependency>
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-sql_2.10</artifactId>
			<version>2.1.1</version>
		</dependency>
		<dependency>
			<groupId>org.scala-lang</groupId>
			<artifactId>scala-library</artifactId>
			<version>2.13.0-M1</version>
		</dependency>
		<dependency>
			<groupId>org.apache.parquet</groupId>
			<artifactId>parquet-hadoop-bundle</artifactId>
			<version>1.8.1</version>
		</dependency>
	</dependencies>
</project>







--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org