HDFS FileSystem常用方法

 

  

工程所依赖的jar包， hadoop-1.1.2/*.jar、hadoop-1.1.2/lib/*.jar

 

 编写Java程序通过URL访问HDFS

public class HDFSTest {
	public static final String HDFSPATH ="hdfs://centos:9000/hello.txt";
	@Test
	public void testAcceHDFSForURL() throws IOException{
		URL url = new URL(HDFSPATH);
		InputStream inputStream = url.openStream();
		IOUtils.copyBytes(inputStream, System.out, 1024, true);
	}
}
java.net.MalformedURLException: unknown protocol: hdfs

 只是这样访问，程序会有异常，畸形的URL，就是不能解析HDFS这种协议，使用setURLStreamHandlerFactory()让URL能够识别hdfs协议

public class HDFSTest {
	//URL如果不是用IP地址，使用主机名，需要在hosts文件中配置主机名到IPd地址的映射
	public static final String HDFSPATH ="hdfs://192.168.56.101:9000/hello.txt";
	@Test
	public void testAcceHDFSForURL() throws IOException{
		URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
		URL url = new URL(HDFSPATH);
		InputStream inputStream = url.openStream();
		/**
			InputStream：输入流			OutputStream：输出流			int：缓冲区大小			boolean：传输结束后是否关闭流
		**/
		IOUtils.copyBytes(inputStream, System.out, 1024, true);
	}
}

 

使用HDFS提供的接口访问HDFS，功能更加强大，操作类FileSystem

删除HDFS中的所有文件、目录

[root@centos Downloads]# hadoop fs -rmr hdfs://centos:9000/*
Warning: $HADOOP_HOME is deprecated.
Deleted hdfs://centos:9000/file1
Deleted hdfs://centos:9000/hello.txt
Deleted hdfs://centos:9000/usr

[root@centos Downloads]# hadoop fs -lsr /
Warning: $HADOOP_HOME is deprecated.
[root@centos Downloads]# hadoop fs -lsr /
Warning: $HADOOP_HOME is deprecated.

 

创建一个名为file1的目录

        public static final String HDFSURIPATH ="hdfs://192.168.56.101:9000/";
	@Test
	public void testFileSystemForUploadFile() throws IOException, URISyntaxException{
		FileSystem fileSystem = getFileSystem();
		fileSystem.mkdirs(new Path("/file1"));
	}

	private FileSystem getFileSystem() throws IOException, URISyntaxException {
		//Configuration对象加载配置文件
		FileSystem fileSystem = FileSystem.get(new URI(HDFSURIPATH), new Configuration());
		return fileSystem;
	}

 通过命令查看文件创建成功

[root@centos Downloads]# hadoop fs -lsr /
Warning: $HADOOP_HOME is deprecated.
drwxr-xr-x   - Administrator supergroup          0 2015-05-22 03:49 /file1

 

删除目录

	@Test
	public void testFileSystemForDelete() throws IOException, URISyntaxException{
		FileSystem fileSystem = getFileSystem();
		fileSystem.delete(new Path("/file1"),true);
	}

 命令查看

[root@centos Downloads]# hadoop fs -lsr /
Warning: $HADOOP_HOME is deprecated.

 

上传文件

	@Test
	public void testFileSystemForCreate() throws IOException, URISyntaxException{
		//创建一个名称为file1的目录
		testFileSystemForMkdir();
		//创建一个文件
		FSDataOutputStream fsOutputStream = getFileSystem().create(new Path("/file1/hello.txt"));
		//获取本地文件
		FileInputStream fInputStrea = new FileInputStream("E:/Linux/Linux.txt");
		//把本地文件E:/Linux/Linux.txt中的内容拷贝到hdfs中/file/hello.txt文件中
		IOUtils.copyBytes(fInputStrea, fsOutputStream, 1024, true);
	}

另一种上传文件的方式

FileSystem fs = HDFSUtils.getFileSystem();
Path s = new Path("E:/Linux/201207141451233056.png");
Path d = new Path("/file1/");
fs.copyFromLocalFile(s, d); 

命令查看

[root@centos Downloads]# hadoop fs -lsr /
Warning: $HADOOP_HOME is deprecated.
drwxr-xr-x   - Administrator supergroup          0 2015-05-22 04:14 /file1
-rw-r--r--   3 Administrator supergroup       4617 2015-05-22 04:14 /file1/hello.txt

 

读取文件，中文内容会有乱码

	@Test
	public void testFileSystemForOpen() throws IOException, URISyntaxException{
		 FSDataInputStream fsInputStream = getFileSystem().open(new Path("/file1/hello.txt"));
		IOUtils.copyBytes(fsInputStream, System.out, 1024, true);
	}

 

查看目录及文件

	@Test
	public void testFileSystemForListStatus() throws IOException, URISyntaxException{
		 FileStatus[] listStatus = getFileSystem().listStatus(new Path("/file1"));
		for (FileStatus fs : listStatus) {
			System.out.println(fs.getLen());
			System.out.println(fs.isDir());
			System.out.println(fs.getPath());
			System.out.println(fs.getPermission());
			System.out.println(fs.getReplication());
		}
	}