在Java中減少地圖Hadoop

Question

我是Hadoop的新手。我有以下格式的文件：

123textfinderlater。它是一個固定寬度的文件。我想添加一個分隔符。假設假設我的第一個字段是123，即長度爲3，第二個字段是textfinder，即：長度爲10，第三個字段後面爲ie.length 5.每個字段都有一個預定義的長度。現在我需要添加一個分隔符來分隔我的字段。我的輸出應該是123 | textfinder |稍後。我只有值（文件中的行）。映射器和減速器程序的關鍵應該是什麼。

在此先感謝

Answer 1

你甚至不需要你的具體情況減速，用於映射的鍵值仍然line no. - line像往常一樣，那麼你只需要編寫回線您添加分隔符爲鍵入。檢查以下代碼：

import java.io.IOException; 

import org.apache.hadoop.conf.Configuration; 
import org.apache.hadoop.conf.Configured; 
import org.apache.hadoop.io.NullWritable; 
import org.apache.hadoop.io.LongWritable; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Job; 
import org.apache.hadoop.mapreduce.Mapper; 
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
import org.apache.hadoop.util.Tool; 
import org.apache.hadoop.util.ToolRunner; 



public class Delimiter extends Configured implements Tool { 

    public static class DelimiterMapper 
     extends Mapper<LongWritable, Text, Text, NullWritable> { 

    private static Text addDelimiter(Text value, char delimiter) { 
    String str = value.toString(); 
    String ret = str.substring(0,2) + delimiter + str.substring(3,12) + delimiter + str.substring(13); 
    return new Text(ret); 
    } 

    public void map(LongWritable key, Text value, Context context) 
        throws IOException, InterruptedException { 
     context.write(addDelimiter(value, '|'), NullWritable.get()); 
    } 

    } 

    public int run(String[] args) 
    throws IOException, InterruptedException, ClassNotFoundException { 
    Job job = Job.getInstance(getConf()); 
    if (args.length != 2) { 
     System.err.println("Usage: Delimiter <in> <out>"); 
     return 2; 
    } 

    FileInputFormat.addInputPath(job, new Path(args[0])); 
    Path outputDir = new Path(args[1]); 
    if (outputDir.getFileSystem(getConf()).exists(outputDir)) { 
     throw new IOException("Output directory " + outputDir + 
           "already exists"); 
    } 
    FileOutputFormat.setOutputPath(job, outputDir); 
    job.setJobName("Delimiter"); 
    job.setJarByClass(Delimiter.class); 
    job.setMapperClass(DelimiterMapper.class); 
    job.setNumReduceTasks(0); 
    job.setOutputKeyClass(Text.class); 
    job.setOutputValueClass(NullWritable.class); 
    return job.waitForCompletion(true) ? 0:1; 

    } 

    public static void main(String[] args) throws Exception { 
    int res = ToolRunner.run(new Configuration(), new Delimiter(), args); 
    System.exit(res); 
    } 
}