星火1.3.1 SQL libriary：scala.reflect.internal.MissingRequirementError：類org.apache.spark.sql.catalyst.ScalaReflection

Question

當嘗試運行這段代碼：

val conf = new SparkConf() 
    .setMaster("local[1]") 
    .setAppName("Small") 
    .set("spark.executor.memory", "2g") 
val sc = new SparkContext(conf) 

val sqlContext = new org.apache.spark.sql.SQLContext(sc) 
import sqlContext.implicits._ 
val df = sc.parallelize(Array((1,30),(2,10),(3,20),(1,10)(2,30))).toDF("books","readers") 
val results = df.join(
df.select($"books" as "r_books", $"readers" as "r_readers"), 
$"readers" === $"r_readers" and $"books" < $"r_books" 
) 
.groupBy($"books", $"r_books") 
.agg($"books", $"r_books", count($"readers")) 

在SBT控制檯開始與以下build.sbt：

name := "Small" 

version := "1.0" 

scalaVersion := "2.10.4" 

libraryDependencies += "org.apache.spark" %% "spark-core" % "1.3.1" 

libraryDependencies += "org.apache.spark" %% "spark-sql" % "1.3.1" 

返回錯誤：

scala.reflect.internal.MissingRequirementError：類org.apache.spark.sql.catalyst.ScalaReflection在JavaMirror與java.net.URLClass Loader @ 13a9a4f9 ...

任何想法？

Answer 1

你有一些拼寫錯誤，最後的最後一個函數計數不能有您需要導入org.apache.spark.sql.functions._

檢查這一點，我CORRCT拼寫錯誤，並增加進口

import org.apache.spark.{ SparkContext, SparkConf } 
import org.apache.spark.sql.functions._ 

/** 
* Created by anquegi on 01/06/15. 
*/ 
object QSpark162015 extends App { 

    val conf = new SparkConf() 
    .setMaster("local[2]") 
    .setAppName("QSpark162015") 
    .set("spark.executor.memory", "2g") 

    val sc = new SparkContext(conf) 

    val sqlContext = new org.apache.spark.sql.SQLContext(sc) 

    import sqlContext.implicits._ 

    val df = sc.parallelize(Array((1, 30), (2, 10), (3, 20), (1, 10), (2, 30))).toDF("books", "readers") 
    val results = df.join(
    df.select($"books" as "r_books", $"readers" as "r_readers"), 
    $"readers" === $"r_readers" and $"books" < $"r_books" 
) 
    .groupBy($"books", $"r_books") 
    .agg($"books", $"r_books", count($"readers")) 

    results.foreach(println _) 

    sc.stop() 

}