|
u Îļþ²Ù×÷
Ø ÉÏ´«±¾µØÎļþµ½HDFS
Ø ¶ÁÈ¡Îļþ
Ø ÔÚhadoop fsÖÐн¨Îļþ£¬²¢Ð´Èë
Ø ÖØÃüÃûÎļþ
Ø É¾³ýHadoop fsÉϵÄÎļþ
u Ŀ¼²Ù×÷
Ø ¶Áȡij¸öĿ¼ÏµÄËùÓÐÎļþ
Ø ÔÚhadoop fsÉÏ´´½¨Ä¿Â¼
Ø É¾³ýĿ¼
u HDFSÐÅÏ¢
Ø ²éÕÒij¸öÎļþÔÚHDFS¼¯ÈºµÄλÖÃ
Ø »ñÈ¡HDFS¼¯ÈºÉÏËùÓнڵãÃû³ÆÐÅÏ¢
Ïêϸ´úÂëÈçÏ£º
package org.ch.hadoop.hdfs;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Hdfs;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.io.IOUtils;
import org.ch.hadoop.hdfs.Util.HDFSUtils;
import org.junit.Test;
import org.mockito.internal.matchers.EndsWith;
public class HDFSFStest {
/*
* ¶ÁÈ¡ÎļþÄÚÈÝ
* **/
@Test
public void testRead() throws Exception{
//»ñÈ¡Îļþϵͳ
FileSystem hdfs= HDFSUtils.getFileSystem();
//ÎļþÃû³Æ
Path path = new Path("/input/b.txt");
//´ò¿ªÎļþÊäÈëÁ÷
FSDataInputStream inputStream = hdfs.open(path);
//¶ÁÈ¡ÎļþÄÚÈݵ½¿ØÖÆ̨ÏÔʾ ----read()
IOUtils.copyBytes(inputStream, System.out,4096,false);
//¹Ø±ÕÁ÷ ------close()
IOUtils.closeStream(inputStream);
}
/*
*²é¿´Ä¿Â¼
**/
@Test
public void testList() throws Exception{
FileSystem hdfs = HDFSUtils.getFileSystem();
//Ŀ¼
Path path = new Path("/input");
FileStatus[] fileStatus = hdfs.listStatus(path);
for(FileStatus fStatus : fileStatus){
Path p = fStatus.getPath();
String info = fStatus.isDirectory()? "Ŀ¼" : "Îļþ";
System.out.println(info + ":" + p);
}
}
/*
*´´½¨Ä¿Â¼
**/
@Test
public void testDirectory() throws IOException{
FileSystem hdfs = HDFSUtils.getFileSystem();
Path path = new Path("/test/dir");
boolean isSucces = hdfs.mkdirs(path);//Ï൱ÓÚ -mkdir -p
String info = isSucces ? "³É¹¦" : "ʧ°Ü";
System.out.println("´´½¨Ä¿Â¼[" + path + "]" + info);
}
/*
* ÉÏ´«Îļþ ----put copyFromLcal
* */
@Test
public void testPut() throws IOException{
FileSystem hdfs = HDFSUtils.getFileSystem();
//ÉÏ´«Ô·¾
Path srcpath= new Path("/home/ch-hduser/myfile/test1.data");
//HDFS ÎļþÉÏ´«Â·¾¶
Path dstpath = new Path("/test/dir");
//ÎļþÉÏ´«
hdfs.copyFromLocalFile(srcpath, dstpath);
}
/*
* ´´½¨HDFSÎļþ²¢Ð´ÈëÄÚÈÝ
* */
@Test
public void testCreate() throws IOException{
FileSystem hdfs = HDFSUtils.getFileSystem();
//HDFS ÎļþÉÏ´«Â·¾¶
Path path = new Path("/test/dir/touch.data");
FSDataOutputStream fsDataOutputStream = hdfs.create(path);
fsDataOutputStream.writeUTF("²âÊÔwrite");
IOUtils.closeStream(fsDataOutputStream);
}
//¶ÔHDFSÉϵÄÎļþ½øÐÐÖØÃüÃû
@Test
public void testRename() throws IOException{
FileSystem hdfs = HDFSUtils.getFileSystem();
//HDFSÎļþÉÏ´«Â·¾¶
Path srcpath = new Path("/test/dir/test1.data");
Path destpath = new Path("/test/dir/rename.data");
boolean flag = hdfs.rename(srcpath, destpath);
System.out.println(flag);
}
//ɾ³ýÎļþ
@Test
public void testDelete() throws IOException{
FileSystem hdFileSystem = HDFSUtils.getFileSystem();
//HDFSÎļþÉÏ´«Â·¾¶
Path path = new Path("/test/dir/rename.data");
boolean flag = hdFileSystem.deleteOnExit(path);
System.out.println(flag);
}
//ɾ³ýĿ¼
@Test
public void testDeleteDir() throws IOException{
FileSystem hdFileSystem = HDFSUtils.getFileSystem();
//HDFSÎļþÉÏ´«Â·¾¶
Path path = new Path("/test/dir");
boolean flag = hdFileSystem.delete(path, true);
System.out.println(flag);
}
//²éÕÒij¸öÎļþÔÚhadoop¼¯ÈºÖеÄλÖÃ
@Test
public void testLocation() throws IOException{
FileSystem hdFileSystem = HDFSUtils.getFileSystem();
//HDFSÎļþÉÏ´«Â·¾¶
Path path = new Path("/input/a.txt");
FileStatus fileStatus = hdFileSystem.getFileStatus(path);
BlockLocation[] blockLocations = hdFileSystem.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
for(BlockLocation blockLocation : blockLocations)
{
String[] hoStrings = blockLocation.getHosts();
for(String hoString : hoStrings)
{
System.out.print(hoString);
}
System.out.println();
}
}
// »ñÈ¡¼¯ÈºÉÏËùÓнڵãÐÅÏ¢
@Test
public void testCluster() throws IOException{
FileSystem hdFileSystem = HDFSUtils.getFileSystem();
//ת»»³É·Ö²¼Ê½Îļþϵͳ
DistributedFileSystem distributedFileSystem = (DistributedFileSystem) hdFileSystem;
//»ñÈ¡½Úµã״̬ÐÅÏ¢
DatanodeInfo[] datanodeInfos = distributedFileSystem.getDataNodeStats();
//¸ù¾Ý½ÚµãÐÅÏ¢´òÓ¡³öÖ÷»úÃû
for(DatanodeInfo datanodeInfo : datanodeInfos)
{
String hoString = datanodeInfo.getHostName();
System.out.println(hoString);
}
}
}