收藏一个java操作HDFS的工具类,比较简单
By:Roy.LiuLast updated:2019-10-24
收藏一个java操作HDFS的工具类,比较简单, 偶尔应急的时候,拿出来参考下。
org.apache.hadoop hadoop-client 2.7.7
public class HDFSUtil { public static final String HDFS_PATH = "hdfs://192.168.99.228:9000"; FileSystem fileSystem = null; Configuration configuration = null; /** * 创建HDFS目录 */ public void mkdir() throws Exception { fileSystem.mkdirs(new Path("/hdfsapi/test")); } /** * 创建文件 */ public void create() throws Exception { FSDataOutputStream output = fileSystem.create(new Path("/hdfsapi/test/a.txt")); output.write("hello hadoop".getBytes()); output.flush(); output.close(); } /** * 查看HDFS文件的内容 */ public void cat() throws Exception { FSDataInputStream in = fileSystem.open(new Path("/hdfsapi/test/a.txt")); IOUtils.copyBytes(in, System.out, 1024); in.close(); } /** * 重命名 */ public void rename() throws Exception { Path oldPath = new Path("/hdfsapi/test/a.txt"); Path newPath = new Path("/hdfsapi/test/b.txt"); fileSystem.rename(oldPath, newPath); } /** * 上传文件到HDFS * * @throws Exception */ public void copyFromLocalFile() throws Exception { Path localPath = new Path("/Users/rocky/data/hello.txt"); Path hdfsPath = new Path("/hdfsapi/test"); fileSystem.copyFromLocalFile(localPath, hdfsPath); } /** * 上传文件到HDFS */ public void copyFromLocalFileWithProgress() throws Exception { InputStream in = new BufferedInputStream( new FileInputStream( new File("/Users/rocky/source/spark-1.6.1/spark-1.6.1-bin-2.6.0-cdh5.5.0.tgz"))); FSDataOutputStream output = fileSystem.create(new Path("/hdfsapi/test/spark-1.6.1.tgz"), new Progressable() { public void progress() { System.out.print("."); //带进度提醒信息 } }); IOUtils.copyBytes(in, output, 4096); } /** * 下载HDFS文件 */ public void copyToLocalFile() throws Exception { Path localPath = new Path("/Users/rocky/tmp/h.txt"); Path hdfsPath = new Path("/hdfsapi/test/hello.txt"); fileSystem.copyToLocalFile(hdfsPath, localPath); } /** * 查看某个目录下的所有文件 */ public void listFiles() throws Exception { FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/")); for(FileStatus fileStatus : fileStatuses) { String isDir = fileStatus.isDirectory() ? "文件夹" : "文件"; short replication = fileStatus.getReplication(); long len = fileStatus.getLen(); String path = fileStatus.getPath().toString(); System.out.println(isDir + "\t" + replication + "\t" + len + "\t" + path); } } /** * 删除 */ public void delete() throws Exception{ fileSystem.delete(new Path("/"), true); } public void setUp() throws Exception { System.out.println("HDFSApp.setUp"); configuration = new Configuration(); fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "summer"); } public void tearDown() throws Exception { configuration = null; fileSystem = null; System.out.println("HDFSApp.tearDown"); } }
From:一号门
Previous:利用python ipset来设置linux防火墙黑白名单的方法
Next:获取java对象地址的一个工具类
COMMENTS