hadoop报错：　java.lang.ArrayIndexOutOfBoundsException

每次map跑到一半的时候就挂掉了
16/04/06 11:48:18 INFO mapred.MapTask: Starting flush of map output
16/04/06 11:48:18 INFO mapred.MapTask: Spilling map output
16/04/06 11:48:18 INFO mapred.MapTask: bufstart = 0; bufend = 201884; bufvoid = 104857600
16/04/06 11:48:18 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 26194704(104778816); length = 19693/6553600
16/04/06 11:48:19 INFO mapred.MapTask: Finished spill 0
16/04/06 11:48:19 INFO mapred.LocalJobRunner: map task executor complete.
16/04/06 11:48:19 WARN mapred.LocalJobRunner: job_local73117192_0001
java.lang.Exception: java.lang.ArrayIndexOutOfBoundsException: 2
      at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
      at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:522)
Caused by: java.lang.ArrayIndexOutOfBoundsException: 2
      at cn.kxl.identicalPassword$mymapper.map(identicalPassword.java:26)
      at cn.kxl.identicalPassword$mymapper.map(identicalPassword.java:1)
      at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:146)
      at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
      at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
      at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:243)
      at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
      at java.util.concurrent.FutureTask.run(FutureTask.java:262)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
      at java.lang.Thread.run(Thread.java:745)
16/04/06 11:48:19 INFO mapreduce.Job: Job job_local73117192_0001 failed with state FAILED due to: NA
16/04/06 11:48:19 INFO mapreduce.Job: Counters: 0

百度提出解决方法:

custom param中加入 mapred.ifile.buffer.reset.size.mb:256不知道怎么用.............

神豪。 · 发表于 2016-4-6 19:08:33

我是使用的是: eclipse编码然后导出成jar放到集群中运行的。这个错误不知道怎么搞

atsky123 · 发表于 2016-4-6 19:48:31

神豪。发表于 2016-4-6 19:08
我是使用的是: eclipse编码然后导出成jar放到集群中运行的。这个错误不知道怎么搞

在开发环境中是否有这个错误

Alkaloid0515 · 发表于 2016-4-6 19:50:31

这是数组越界，先确保自己的程序源码没有问题

神豪。 · 发表于 2016-4-6 19:51:44

Alkaloid0515 发表于 2016-4-6 19:50
这是数组越界，先确保自己的程序源码没有问题

源码

[mw_shl_code=java,true]package cn.kxl;

import java.io.IOException;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class identicalPassword {

public static class mymapper extends Mapper<LongWritable, Text, Text, LongWritable> {
private static final LongWritable one=new LongWritable(1);
private Text mailbox = new Text();
private Text password = new Text();
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] lins=StringUtils.split(value.toString(),",");
if(lins[2].length()>0&&lins[3].length()>0){
password.set(lins[2]);
mailbox.set(lins[3]);
System.err.println("----"+password+"\t "+mailbox);
context.write(password,one);
}
}
}

public static class myreduce extends Reducer<Text, LongWritable, Text, LongWritable> {
private LongWritable result = new LongWritable();

public void reduce(Text key, Iterable<LongWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (LongWritable val : values) {
sum+=val.get();
}
result.set(sum);
context.write(key, result);
}

}

public static void main(String[] args) throws Exception, IOException {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length < 2) {
System.err.println(" input and ouput");
System.exit(2);
}
Job job = Job.getInstance(conf, "word count");
job.setJarByClass(independentIp.class);

job.setMapperClass(mymapper.class);
job.setReducerClass(myreduce.class);

job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);

job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);

for (int i = 0; i < otherArgs.length - 1; i++) {
FileInputFormat.addInputPath(job, new Path(otherArgs));
}
FileOutputFormat.setOutputPath(job, new Path(otherArgs[(otherArgs.length-1)]));

System.exit(job.waitForCompletion(true)?0:1);
}

}
[/mw_shl_code]