hadoop編譯

我是hadoop的新手。我從網上hadoop編譯

import java.io.IOException; 
import java.util.*; 

import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.conf.*; 
import org.apache.hadoop.io.*; 
import org.apache.hadoop.mapred.*; 
import org.apache.hadoop.util.*; 

public class Gender { 

    private static String genderCheck = "female"; 

    public static class Map extends MapReduceBase implements Mapper { 
     private final static IntWritable one = new IntWritable(1); 
     private Text locText = new Text(); 

     public void map(LongWritable key, Text value, OutputCollector output, Reporter reporter) throws IOException { 
      String line = value.toString(); 
      String location = line.split(",")[14] + "," + line.split(",")[15]; 
      long male = 0L; 
      long female = 0L; 
      if (line.split(",")[17].matches("\d+") && line.split(",")[18].matches("\d+")) { 
       male = Long.parseLong(line.split(",")[17]); 
       female = Long.parseLong(line.split(",")[18]); 
      } 
      long diff = male - female; 
      locText.set(location); 
      if (Gender.genderCheck.toLowerCase().equals("female") && diff < 0) { 
       output.collect(locText, new LongWritable(diff * -1L)); 
      } 
      else if (Gender.genderCheck.toLowerCase().equals("male") && diff 
> 0) { 
       output.collect(locText, new LongWritable(diff)); 
      } 
     } } 

    public static void main(String[] args) throws Exception { 
     JobConf conf = new JobConf(Gender.class); 
     conf.setJobName("gender"); 
     conf.setOutputKeyClass(Text.class); 
     conf.setOutputValueClass(LongWritable.class); 
     conf.setMapperClass(Map.class); 

     if (args.length != 3) { 
      System.out.println("Usage:"); 
      System.out.println("[male/female] /path/to/2kh/files /path/to/output"); 
      System.exit(1); 
     } 

     if (!args[0].equalsIgnoreCase("male") && !args[0].equalsIgnoreCase("female")) { 
      System.out.println("first argument must be male or female"); 
      System.exit(1); 
     } 
     Gender.genderCheck = args[0]; 

     conf.setInputFormat(TextInputFormat.class); 
     conf.setOutputFormat(TextOutputFormat.class); 
     FileInputFormat.setInputPaths(conf, new Path(args[1])); 
     FileOutputFormat.setOutputPath(conf, new Path(args[2])); 
     JobClient.runJob(conf); } 

}

此代碼時，我使用「的javac -cp /usr/local/hadoop/hadoop-core-1.0.3.jar Gender.java」

得到以下編譯這段代碼錯誤：

如何正確編譯它？

來源

2013-04-08 Robin

更改類Maper類聲明如下：

public static class Map extends MapReduceBase implements Mapper<LongWritable,Text,Text, LongWritable>

如果不指定任何特定的類名，你就需要有地圖的功能如下：

@Override 
public void map(Object arg0, Object arg1, OutputCollector arg2, Reporter arg3) throws IOException { 
    // TODO Auto-generated method stub 
}

現在，這裏的特定類型表示預期的輸入鍵值對類型和來自映射器的輸出鍵值類型。

在你的情況下，輸入鍵值對是LongWritable-Text。

而且，從output.collect方法調用中猜測，您的映射器輸出鍵值對是Text-LongWritable。

因此，您的地圖類應執行Mapper<LongWritable,Text,Text, LongWritable>。

有在代碼中多了一個錯誤 -

使用"\d+"將無法編譯爲\d沒有意義，反斜線後，預計一個特殊的轉義序列，所以我想對你下面應該工作： line.split(",")[17].matches("\\d+")

來源

2013-04-08 07:07:41 Amar

更改地圖類，如下所示：

public static class Map extends MapReduceBase implements Mapper <Input key, Input value, Output Key , Output Value>

在你的情況下，輸入鍵是LongWritable，輸入值是Text，輸出密鑰是Text，輸出值是LongWritable

public static class Map extends MapReduceBase implements Mapper <LongWritable, Text, Text,LongWritable>

來源

2017-09-12 07:22:49

回答

相關問題