如何運行的火花應用

紗日誌

紗應用-list

紗應用-kill

我試圖命令紗紗線命令使用yarnclient但面臨的問題。

package com.ankush 
import java.io.Serializable 

import org.apache.hadoop.yarn.api.records.ApplicationId 
import org.apache.hadoop.yarn.client.api.YarnClient 
import org.apache.hadoop.yarn.conf._ 
import org.apache.spark._ 
import org.apache.spark.streaming._ 

object SparkFiller extends Serializable{ 
    private var appId: ApplicationId = null 
    def main(args: Array[String]): Unit ={ 

    try { 

    var counter =0 

    val sparkConf = new SparkConf().setAppName("SparkFiller")//.setMaster("local[*]") 
    val conf: YarnConfiguration = new YarnConfiguration() 

    // println("test") 
    sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") 
    sparkConf.set("spark.streaming.stopGracefullyOnShutdown", "true") 
    val ssc = new StreamingContext(sparkConf, Seconds(1)) 
    val lines = ssc.socketTextStream("localhost", 9999) 
    lines.foreachRDD(
    rdd => { 
     if(rdd.isEmpty()) 
     { 
     counter+=1 
     if(counter>4) 
     { 
      println("Testing") 
      val conf: YarnConfiguration = new YarnConfiguration() 
      val client = YarnClient.createYarnClient() 
      client.init(conf) 
      client.start() 
      val newApp = client.createApplication() 
      client.getApplications() 

      val test = client.getApplications() 
      println(test) 

      counter=0 
      //  val result = "sh /home/ankush/test.sh".! 
      //    ssc.stop() 
     } 
     } 

     rdd.foreachPartition(part => { 
     // val dbsender = new PushToDocumentDB(); 
     part.foreach(msg =>{ 
      println(msg) 
     }) 

     }) 
    }) 



    ssc.start() 

    ssc.awaitTermination() 


} 
catch { 

    case runtime: RuntimeException=> { 
    1 
    } 
} 


    } 

}

像

Exception in thread "streaming-job-executor-13" java.lang.NoClassDefFoundError: com/sun/jersey/api/client/config/ClientConfig 
    at org.apache.hadoop.yarn.client.api.TimelineClient.createTimelineClient(TimelineClient.java:59) 
    at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.createTimelineClient(YarnClientImpl.java:187) 
    at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.serviceInit(YarnClientImpl.java:162) 
    at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)

然後現在所面臨的錯誤使用這個應用程序正在試圖做，如果有流經插座一定時間內沒有數據我想列出對紗線的應用和殺死將要發送數據到套接字的另一個應用程序。

謝謝，

Ankush雷迪。

來源

2017-09-14 ankush reddy

澤西jar在類路徑中丟失，添加jersey-bundle jar spark-submit。

spark-submit --jars jersey-bundle-x.y.z.jar \ 
...........

或

spark-submit --conf "spark.executor.extraClassPath=\\<path>jersey-bundle-x.y.z.jar" \ 
.......

來源

2017-09-15 02:22:18

我已經在pom.xml包括它並能夠看到它在罐子裏我與應用程序罐子沿着建設它。 –

它工作還是失敗？ –

它仍然失敗，同樣的錯誤。 –

如何運行的火花應用

回答

相關問題