2
我有一個使用SparkContext的簡單單元測試。我能夠在IntelliJ Idea中運行單元測試，沒有任何問題。然而試圖運行從SBT殼我得到以下錯誤同樣的測試時：無法在SBT中運行Apache Spark相關單元測試 - NoClassDefFoundError
java.lang.NoClassDefFoundError: org/eclipse/jetty/server/bio/SocketConnector
at org.apache.spark.HttpServer.org$apache$spark$HttpServer$$doStart(HttpServer.scala:74)
at org.apache.spark.HttpServer$$anonfun$1.apply(HttpServer.scala:60)
at org.apache.spark.HttpServer$$anonfun$1.apply(HttpServer.scala:60)
at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1446)
at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1442)
at org.apache.spark.HttpServer.start(HttpServer.scala:60)
at org.apache.spark.HttpFileServer.initialize(HttpFileServer.scala:45)
at org.apache.spark.SparkEnv$.create(SparkEnv.scala:243)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:203)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:118)
at com.company.myservice.merging.Merger.<init>(Merger.scala:16)
at com.company.myservice.merging.MergerTest.<init>(MergerTest.scala:16)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at java.lang.Class.newInstance(Class.java:374)
at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:641)
at sbt.TestRunner.runTest$1(TestFramework.scala:84)
at sbt.TestRunner.run(TestFramework.scala:94)
at sbt.TestFramework$$anon$2$$anonfun$$init$$1$$anonfun$apply$8.apply(TestFramework.scala:224)
at sbt.TestFramework$$anon$2$$anonfun$$init$$1$$anonfun$apply$8.apply(TestFramework.scala:224)
at sbt.TestFramework$.sbt$TestFramework$$withContextLoader(TestFramework.scala:212)
at sbt.TestFramework$$anon$2$$anonfun$$init$$1.apply(TestFramework.scala:224)
at sbt.TestFramework$$anon$2$$anonfun$$init$$1.apply(TestFramework.scala:224)
at sbt.TestFunction.apply(TestFramework.scala:229)
at sbt.Tests$.sbt$Tests$$processRunnable$1(Tests.scala:211)
at sbt.Tests$$anonfun$makeSerial$1.apply(Tests.scala:217)
at sbt.Tests$$anonfun$makeSerial$1.apply(Tests.scala:217)
at sbt.std.Transform$$anon$3$$anonfun$apply$2.apply(System.scala:45)
at sbt.std.Transform$$anon$3$$anonfun$apply$2.apply(System.scala:45)
at sbt.std.Transform$$anon$4.work(System.scala:64)
at sbt.Execute$$anonfun$submit$1$$anonfun$apply$1.apply(Execute.scala:237)
at sbt.Execute$$anonfun$submit$1$$anonfun$apply$1.apply(Execute.scala:237)
at sbt.ErrorHandling$.wideConvert(ErrorHandling.scala:18)
at sbt.Execute.work(Execute.scala:244)
at sbt.Execute$$anonfun$submit$1.apply(Execute.scala:237)
at sbt.Execute$$anonfun$submit$1.apply(Execute.scala:237)
at sbt.ConcurrentRestrictions$$anon$4$$anonfun$1.apply(ConcurrentRestrictions.scala:160)
at sbt.CompletionService$$anon$2.call(CompletionService.scala:30)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
這是我build.scala是什麼樣子：
import sbt._
import Keys._
import org.scalatra.sbt._
import org.scalatra.sbt.PluginKeys._
import com.mojolly.scalate.ScalatePlugin._
import ScalateKeys._
import com.github.retronym.SbtOneJar
import com.earldouglas.xsbtwebplugin.PluginKeys._
import com.earldouglas.xsbtwebplugin.WebPlugin._
import sbtassembly.Plugin._
import AssemblyKeys._
import sbtassembly.Plugin.AssemblyKeys._
object myservice extends Build {
val Organization = "com.company"
val Name = "myservice"
val Version = "0.1.0-SNAPSHOT"
val ScalaVersion = "2.10.4"
val ScalatraVersion = "2.3.0"
// settings for sbt-assembly plugin
val myAssemblySettings = assemblySettings ++ Seq(
// handle conflicts during assembly task
mergeStrategy in assembly <<= (mergeStrategy in assembly) {
(old) => {
case PathList("javax", "servlet", xs @ _*) => MergeStrategy.first
case PathList(ps @ _*) if ps.last endsWith ".html" => MergeStrategy.first
case "application.conf" => MergeStrategy.concat
case "unwanted.txt" => MergeStrategy.discard
case x => old(x)
}
},
// copy web resources to /webapp folder
resourceGenerators in Compile <+= (resourceManaged, baseDirectory) map {
(managedBase, base) =>
val webappBase = base/"src"/"main"/"webapp"
for {
(from, to) <- webappBase ** "*" x rebase(webappBase, managedBase/"main"/"webapp")
} yield {
Sync.copy(from, to)
to
}
}
)
lazy val project = Project (
"myapplication",
file("."),
settings = Defaults.defaultSettings ++ ScalatraPlugin.scalatraWithJRebel ++
SbtOneJar.oneJarSettings ++
scalateSettings ++ Seq(
organization := Organization,
name := Name,
version := Version,
scalaVersion := ScalaVersion,
port in container.Configuration := 8000,
resolvers ++= Seq (Classpaths.typesafeReleases,
"Typesafe Releases" at "http://repo.typesafe.com/typesafe/releases/",
"Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots"),
libraryDependencies ++= Seq(
"org.scala-lang" % "scala-reflect" % ScalaVersion,
"org.scala-lang" % "scala-compiler" % ScalaVersion,
"org.scalatest" %% "scalatest" % "2.2.1" % "test",
"org.scalaz" %% "scalaz-core" % "7.0.6",
"org.scalatra" %% "scalatra" % ScalatraVersion,
"org.scalatra" %% "scalatra-scalate" % ScalatraVersion,
"org.scalatra" %% "scalatra-specs2" % ScalatraVersion % "test",
"ch.qos.logback" % "logback-classic" % "1.0.6" % "runtime",
"commons-codec" % "commons-codec" % "1.8",
"org.apache.commons" % "commons-lang3" % "3.3.1",
"org.scalaj" %% "scalaj-http" % "0.3.16",
"net.liftweb" %% "lift-json" % "2.6-M4",
"org.eclipse.jetty" % "jetty-webapp" % "8.1.8.v20121106" % "container;compile",
"org.eclipse.jetty.orbit" % "javax.servlet" % "3.0.0.v201112011016" % "container;provided;test" artifacts (Artifact("javax.servlet", "jar", "jar")),
"com.amazonaws" % "aws-java-sdk" % "1.8.5",
"org.scala-lang.modules" %% "scala-async" % "0.9.1",
"com.typesafe" % "config" % "1.2.1",
"org.scalatra" %% "scalatra-auth" % "2.3.0",
// RDBMS-Mysql
"com.typesafe.slick" %% "slick" % "2.1.0",
"mysql" % "mysql-connector-java" % "latest.release",
"com.h2database" % "h2" % "1.3.175",
// Geolocation
"com.javadocmd" % "simplelatlng" % "1.3.0",
// Spark and Mllib
"org.apache.spark" %% "spark-core" % "1.1.0",
"org.apache.spark" %% "spark-mllib" % "1.1.0",
// Lucene
"org.apache.lucene" % "lucene-core" % "4.8.1",
// for Porter Stemmer
"org.apache.lucene" % "lucene-analyzers-common" % "4.8.1",
// Guava for the dictionary
"com.google.guava" % "guava" % "17.0",
// CSV lib
"com.github.tototoshi" %% "scala-csv" % "1.1.0-SNAPSHOT"
),
scalateTemplateConfig in Compile <<= (sourceDirectory in Compile){ base =>
Seq(
TemplateConfig(
base/"webapp"/"WEB-INF"/"templates",
Seq.empty, /* default imports should be added here */
Seq(
Binding("context", "_root_.org.scalatra.scalate.ScalatraRenderContext", importMembers = true, isImplicit = true)
), /* add extra bindings here */
Some("templates")
)
)
}
)
).settings(myAssemblySettings:_*)
.settings(parallelExecution in ThisBuild := false : _*)
}
我嘗試添加以下所有依賴關係我沒有成功build.scala：
"org.eclipse.jetty" % "jetty-http" % "9.2.1.v20140609" % "container;test",
"org.eclipse.jetty" % "jetty-plus" % "9.2.1.v20140609" % "container;test",
"org.eclipse.jetty" % "jetty-io" % "9.2.1.v20140609" % "container;test",
"org.eclipse.jetty" % "jetty-security" % "9.2.1.v20140609" % "container;test",
"org.eclipse.jetty" % "jetty-server" % "9.2.1.v20140609" % "container;test",
"org.eclipse.jetty" % "jetty-servlet" % "9.2.1.v20140609" % "container;test",
"org.eclipse.jetty" % "jetty-webapp" % "9.2.1.v20140609" % "container;test",
"org.eclipse.jetty" % "jetty-util" % "9.2.1.v20140609" % "container;test",
我試圖讓使用火花1.2.0源的IntelliJ測試設置，但我堅持讓LocalSparkContext和朋友的進口。隨附的源代碼（在我的情況下爲spark-assembly-1.2.0-hadoop2.0.0-mr1-cdh4.2.0.jar）的jar沒有測試源。任何幫助，將不勝感激！ – 2015-01-20 15:17:03