線程「main」中的異常java.lang.NoClassDefFoundError：org/apache/spark/internal/Logging

My Spark Consumer失敗並顯示「logging」錯誤。我發現瀏覽錯誤是由於罐子不兼容。我使用的Spark 1.6.3和所有的依賴關係在POM中使用，XML是1.6.3。我仍然得到同樣的錯誤。下面是我的maven結構和執行時拋出錯誤。線程「main」中的異常java.lang.NoClassDefFoundError：org/apache/spark/internal/Logging

Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/spark/internal/Logging 
    at java.lang.ClassLoader.defineClass1(Native Method) 
    at java.lang.ClassLoader.defineClass(ClassLoader.java:763) 
    at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) 
    at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) 
    at java.net.URLClassLoader.access$100(URLClassLoader.java:73) 
    at java.net.URLClassLoader$1.run(URLClassLoader.java:368) 
    at java.net.URLClassLoader$1.run(URLClassLoader.java:362) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at java.net.URLClassLoader.findClass(URLClassLoader.java:361) 
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424) 
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331) 
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357) 
    at org.apache.spark.streaming.kafka.KafkaUtils$.createStream(KafkaUtils.scala:91) 
    at org.apache.spark.streaming.kafka.KafkaUtils$.createStream(KafkaUtils.scala:66) 
    at org.apache.spark.streaming.kafka.KafkaUtils$.createStream(KafkaUtils.scala:130) 
    at org.apache.spark.streaming.kafka.KafkaUtils.createStream(KafkaUtils.scala) 
    at datapipeline.SparkConsumer.main(SparkConsumer.java:60) 
Caused by: java.lang.ClassNotFoundException: org.apache.spark.internal.Logging 
    at java.net.URLClassLoader.findClass(URLClassLoader.java:381) 
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424) 
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)

我的pom.xml：

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 
    <modelVersion>4.0.0</modelVersion> 
    <groupId>com.iot.app.kafka</groupId> 
    <artifactId>iotanalytics</artifactId> 
    <!-- <packaging>jar</packaging> --> 
    <version>1.0.0</version> 
    <name>IoT Kafka Producer</name> 


    <dependencies> 
     <dependency> 
      <groupId>org.apache.kafka</groupId> 
      <artifactId>kafka-clients</artifactId> 
      <version>0.10.0.0</version> 
     </dependency> 


     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-network-common_2.10 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-network-common_2.10</artifactId> 
      <version>1.6.3</version> 
     </dependency> 



     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka-0-10_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-streaming-kafka-0-10_2.11</artifactId> 
      <version>2.0.0</version> 
     </dependency> 



     <dependency> 
      <groupId>org.apache.avro</groupId> 
      <artifactId>avro</artifactId> 
      <version>1.8.0</version> 
     </dependency> 

     <dependency> 
      <groupId>com.fasterxml.jackson.core</groupId> 
      <artifactId>jackson-databind</artifactId> 
      <version>2.8.2</version> 
     </dependency> 

     <dependency> 
     <groupId>com.fasterxml.jackson.module</groupId> 
     <artifactId>jackson-module-scala_2.10</artifactId> 
     <version>2.8.2</version> 
     </dependency> 

     <dependency> 
      <groupId>com.fasterxml.jackson.core</groupId> 
      <artifactId>jackson-annotations</artifactId> 
      <version>2.8.2</version> 
     </dependency> 

     <dependency> 
      <groupId>com.fasterxml.jackson.core</groupId> 
      <artifactId>jackson-core</artifactId> 
      <version>2.8.2</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/javax.json/javax.json-api --> 
     <dependency> 
      <groupId>javax.json</groupId> 
      <artifactId>javax.json-api</artifactId> 
      <version>1.1</version> 
     </dependency> 



     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming_2.11 --> 

     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-streaming_2.11</artifactId> 
      <version>1.6.3</version> 
      <scope>provided</scope> 
     </dependency> 

     <dependency> 
      <groupId>org.glassfish</groupId> 
      <artifactId>javax.json</artifactId> 
      <version>1.0.4</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client --> 
     <dependency> 
      <groupId>org.apache.hadoop</groupId> 
      <artifactId>hadoop-client</artifactId> 
      <version>2.2.0</version> 
     </dependency> 


     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-core_2.11</artifactId> 
      <version>1.6.3</version> 
      <scope>provided</scope> 
     </dependency> 




     <dependency> 
      <groupId>org.spark-project.spark</groupId> 
      <artifactId>unused</artifactId> 
      <version>1.0.0</version> 
      <scope>provided</scope> 
     </dependency> 




     <dependency> 
      <groupId>log4j</groupId> 
      <artifactId>log4j</artifactId> 
      <version>1.2.17</version> 
     </dependency> 


     <!-- <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> 
      <version>2.11.8</version> </dependency> --> 


     <!-- https://mvnrepository.com/artifact/org.scala-lang/scala-library --> 
     <dependency> 
      <groupId>org.scala-lang</groupId> 
      <artifactId>scala-library</artifactId> 
      <version>2.11.9</version> 
     </dependency> 


     <dependency> 
      <groupId>com.typesafe.akka</groupId> 
      <artifactId>akka-actor_2.11</artifactId> 
      <version>2.3.11</version> 
     </dependency> 


     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka-0-8-assembly_2.11 --> 
<dependency> 
    <groupId>org.apache.spark</groupId> 
    <artifactId>spark-streaming-kafka-0-8-assembly_2.11</artifactId> 
    <version>2.0.0</version> 
</dependency> 



     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-sql_2.10</artifactId> 
      <version>1.6.3</version> 
     </dependency> 

     <dependency> 
      <groupId>junit</groupId> 
      <artifactId>junit</artifactId> 
      <version>4.4</version> 
      <scope>test</scope> 
     </dependency> 



    </dependencies> 


    <properties> 
     <maven.compiler.source>1.8</maven.compiler.source> 
     <maven.compiler.target>1.8</maven.compiler.target> 

    </properties> 

</project>

來源

2017-08-11 user3837415

你有沒有試過保存你的pom.xml，右鍵單擊你的項目，mavan構建？也許它只是沒有更新。 – Nathan

您可以簡單地按Ctrl + Shift + T並搜索「Logging」類，如果您可以看到多個相同的版本，則表示它正在加載您需要的舊一個依賴關係[稱爲傳遞maven依賴關係]通過排除它來解決。 –

@MonisMajeed我也嘗試通過刪除庫中的所有jar來構建新的項目。仍然沒有運氣。我 – user3837415

-1

類org.apache.spark.Logging是Spark版本1.5.2可用或更低不在更高版本。

嘗試降級星火版本1.5.2

<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11 --> 
<dependency> 
    <groupId>org.apache.spark</groupId> 
    <artifactId>spark-core_2.11</artifactId> 
    <version>1.5.2</version> 
    <scope>provided</scope> 
</dependency>

來源

2017-08-11 09:12:10

出於好奇：Spark日誌的更新版本如何？我無法想象他們放棄了伐木？換句話說：不應該有這個庫的替代品與OP所使用的新版火花版本匹配嗎？這聽起來是錯誤的降級整個事情，以使其工作... – GhostCat

你的意思是org.apache.spark.internal.Logging？ – toongeorges

日誌記錄總是存在於Spark中，你們可以將其作爲依賴檢查https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.10 –

有目前的火花的低版本的，我以前BUIL罐子。那些沒有被刷新。我刪除了舊版本的jar並檢查了我的pom.xml並確保所有的依賴關係具有相同的版本並構建了項目。程序成功運行。這裏是我的pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 
    <modelVersion>4.0.0</modelVersion> 
    <groupId>com.iot.app.kafka</groupId> 
    <artifactId>DeviceStreaming</artifactId> 
    <packaging>jar</packaging> 
    <version>1.0.0</version> 
    <name>DeviceStreaming</name> 


    <dependencies> 

     <dependency> 
      <groupId>org.apache.kafka</groupId> 
      <artifactId>kafka-clients</artifactId> 
      <version>0.9.0.0</version> 
     </dependency> 

     <!-- <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-network-common_2.11</artifactId> 
      <version>2.1.0</version> </dependency> --> 


     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-network-common_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-network-common_2.11</artifactId> 
      <version>1.6.3</version> 
     </dependency> 


     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-streaming-kafka_2.11</artifactId> 
      <version>1.6.3</version> 
     </dependency> 





     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-streaming_2.11</artifactId> 
      <version>1.6.3</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.10 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-sql_2.11</artifactId> 
      <version>1.6.3</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/org.glassfish/javax.json --> 
     <dependency> 
      <groupId>org.glassfish</groupId> 
      <artifactId>javax.json</artifactId> 
      <version>1.0.4</version> 
     </dependency> 


     <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client --> 
     <dependency> 
      <groupId>org.apache.hadoop</groupId> 
      <artifactId>hadoop-client</artifactId> 
      <version>2.2.0</version> 
     </dependency> 


     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-core_2.11</artifactId> 
      <version>1.6.3</version> 
     </dependency> 


     <dependency> 
      <groupId>org.spark-project.spark</groupId> 
      <artifactId>unused</artifactId> 
      <version>1.0.0</version> 
      <scope>provided</scope> 
     </dependency> 



     <!-- https://mvnrepository.com/artifact/javax.json/javax.json-api --> 
     <dependency> 
      <groupId>javax.json</groupId> 
      <artifactId>javax.json-api</artifactId> 
      <version>1.1</version> 
     </dependency> 








     <dependency> 
      <groupId>log4j</groupId> 
      <artifactId>log4j</artifactId> 
      <version>1.2.17</version> 
     </dependency> 
     <dependency> 
      <groupId>junit</groupId> 
      <artifactId>junit</artifactId> 
      <version>4.12</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/com.google.guava/guava --> 
     <dependency> 
      <groupId>com.google.guava</groupId> 
      <artifactId>guava</artifactId> 
      <version>14.0.1</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/org.onosproject/onos-api --> 
     <dependency> 
      <groupId>org.onosproject</groupId> 
      <artifactId>onos-api</artifactId> 
      <version>1.6.0</version> 
     </dependency> 



     <dependency> 
      <groupId>org.scala-lang</groupId> 
      <artifactId>scala-library</artifactId> 
      <version>2.11.0</version> 
     </dependency> 




     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-streaming-kafka-0-8-assembly_2.11</artifactId> 
      <version>2.0.0</version> 
     </dependency> 

     <dependency> 
      <groupId>org.apache.avro</groupId> 
      <artifactId>avro</artifactId> 
      <version>1.8.0</version> 
     </dependency> 

     <dependency> 
      <groupId>com.twitter</groupId> 
      <artifactId>bijection-avro_2.10</artifactId> 
      <version>0.9.2</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/com.googlecode.json-simple/json-simple --> 
<dependency> 
    <groupId>com.googlecode.json-simple</groupId> 
    <artifactId>json-simple</artifactId> 
    <version>1.1</version> 
</dependency> 


    </dependencies> 


    <properties> 
     <maven.compiler.source>1.8</maven.compiler.source> 
     <maven.compiler.target>1.8</maven.compiler.target> 

    </properties> 


</project>

來源

2017-08-22 13:46:32 user3837415

線程「main」中的異常java.lang.NoClassDefFoundError：org/apache/spark/internal/Logging

回答

相關問題