HDFS 文件操作测试

 

package com.dj.hadoop;

 

import java.io.BufferedInputStream;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileOutputStream;

import java.io.InputStream;

import java.net.URI;

 

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang.builder.ToStringBuilder;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.util.Progressable;

 

/**

 * @description hdfs文件操作

 * @author dj

 * @date 2012-10-10

 */

public class FileHandlerTest {

 

public static String basePath = "hdfs://192.168.10.166:54310";

public static String fileRootPath = basePath

+ "/home/hadoop/data/hadoopdata";

 

public static void uploadLocalFileToHDFS(File localFile, Path destPath)

throws Exception {

Configuration config = new Configuration();

FileSystem.setDefaultUri(config, new URI(basePath));

FileSystem hdfs = FileSystem.get(config);

// Path dst = new Path(fileRootPath,destPath);

// hdfs.copyFromLocalFile(src, dst);

FSDataOutputStream out = hdfs.create(destPath,new Progressable() {

@Override

public void progress() {

    System.out.println("文件进度");  

}

});

   InputStream in = new BufferedInputStream(new FileInputStream(localFile));  

IOUtils.copy(in, out);

hdfs.close();

}

/**

* 文件下载

* @param destPath

* @param localDir

* @throws Exception 

*/

public static void downloadFileFromHDFS(Path destPath,File localDir) throws Exception {

Configuration config = new Configuration();

FileSystem.setDefaultUri(config, new URI(basePath));

FileSystem hdfs = FileSystem.get(config);

if (hdfs.exists(destPath)) {

FSDataInputStream in = hdfs.open(destPath);

FileStatus stat = hdfs.getFileStatus(destPath);

byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat.getLen()))];

in.readFully(0, buffer);

in.close();

hdfs.close();

IOUtils.write(buffer, new FileOutputStream(localDir+"/"+destPath.getName()));

}

}

 

/**

* 删除文件

* @param destPath

* @throws Exception

*/

public static boolean deleteFile(Path destPath)throws Exception {

Configuration config = new Configuration();

FileSystem.setDefaultUri(config, new URI(basePath));

FileSystem hdfs = FileSystem.get(config);

if (hdfs.exists(destPath)) {

return hdfs.delete(destPath,true);

}

return false;

}

public static void listAll(String dir) throws Exception {

Configuration config = new Configuration();

FileSystem.setDefaultUri(config, new URI(basePath));

FileSystem fs = FileSystem.get(config);

FileStatus[] stats = fs.listStatus(new Path(basePath,dir));

for (int i = 0; stats != null && i < stats.length; ++i) {

// System.out.println(ToStringBuilder.reflectionToString(stats[i]));

if (!stats[i].isDir()) {

// regular file

System.out.println("文件:"+stats[i].getPath().toString()+"===="+stats[i].getGroup());

} else if (stats[i].isDir()) {

// dir

System.out.println("文件夹:"+stats[i].getPath().toString()+"===="+stats[i].getGroup());

}  

}

fs.close();

}

 

public static void createDirectory(String directoryName) throws Exception {

Configuration config = new Configuration();

FileSystem.setDefaultUri(config, new URI(basePath));

FileSystem fs = FileSystem.get(config);

System.out.println(ToStringBuilder.reflectionToString(fs));

fs.mkdirs(new Path(fileRootPath, directoryName));

fs.close();

}

 

public static void deleteDirectory(String directoryName) throws Exception {

Configuration config = new Configuration();

FileSystem.setDefaultUri(config, new URI(basePath));

FileSystem fs = FileSystem.get(config);

fs.delete(new Path(fileRootPath, directoryName),true);

fs.close();

}

 

public static void main(String[] args) throws Exception {

String directoryName = "books";

// 【1】创建文件夹

createDirectory(directoryName);

// 【2】 删除文件夹

// deleteDirectory(directoryName);

//【3】 显示文件

//  listAll("/home/hadoop/data/hadoopdata/");

//【4】 文件上传

File file = new File("G:/apiqianyi/ap2last_tmp.txt");

Path destPath = new Path(fileRootPath,directoryName+"/"+file.getName());

// uploadLocalFileToHDFS(file, destPath);

//【5】文件下载

// downloadFileFromHDFS(destPath, new File("G:/"));

//【6】删除文件

boolean flag = deleteFile(destPath);

System.out.println(flag);

}

 

}