[笔记]hdfs namenode FSImage分析1

元数据文件fsimage的分析
fsimage为元数据镜像文件，存储的是某一时刻NameNode内存元数据信息，包括所有的INode信息、正在写入的文件信息以及其他的一些状态信息等。

引用

  /**
   * Save current image and empty journal into {@code current} directory.
   */
  protected void saveCurrent(StorageDirectory sd) throws IOException

当StorageDirectory的类型为NameNodeFile.IMAGE时，将FSImage持久化为../current/fsimage文件。
fsimage的部分结构分析如下：

int	int	long	long	...
LAYOUT_VERSION	namespaceID	numItemsInTree	timestamp	inodes

inodes为目录时的结构

short	binary	short	long	long	long	int	long	long	permission
namelen	name	0	modificationTime	0	0	-1	nsquota	dsquota	permission

inodes为文件时的结构

short	binary	short	long	long	long	int	...	permission
namelen	name	replication	modificationTime	accessTime	preferredBlockSize	numBlocksInFile	blocks	permission

permission的结构

Text	Text	short
username	groupname	permission

blocks的结构

long	long	long
blockId	numBytes	timestamp

解析某fsimage的结果
LAYOUT_VERSION:-41
namespaceID:1167023747
numItemsInTree:19
generationStamp:1012

node:{name:, modtime:1377679793059, nsquota:2147483647, dsquota:-1, permission:hue supergroup 755}
node:{name:/test, modtime:1377679792720, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/input, modtime:1377679736766, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/output, modtime:1377679792720, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/input/chapter1.txt, copys:2, blksize:67108864, blk:[{3939378972164311396,8976,1002}], modtime:1377679736766, acctime:1377679736766, permission:hue supergroup 644}
node:{name:/test/output/_SUCCESS, copys:2, blksize:67108864, blk:[], modtime:1377679792720, acctime:1377679792720, permission:hue supergroup 644}
node:{name:/test/output/_logs, modtime:1377679773673, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/output/part-r-00000, copys:2, blksize:67108864, blk:[{5401467920515552367,6128,1011}], modtime:1377679787973, acctime:1377679787973, permission:hue supergroup 644}
node:{name:/test/output/_logs/history, modtime:1377679773673, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/output/_logs/history/job_201308281640_0001_1377679773095_hue_word+count, copys:2, blksize:67108864, blk:[{-7275418827867675308,13833,1012}], modtime:1377679773322, acctime:1377679773322, permission:hue supergroup 644}
node:{name:/test/output/_logs/history/job_201308281640_0001_conf.xml, copys:2, blksize:67108864, blk:[{-4116164172580388309,47105,1010}], modtime:1377679773673, acctime:1377679773673, permission:hue supergroup 644}
node:{name:/tmp/hadoop-hue, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred/staging, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred/system, modtime:1377679793049, nsquota:-1, dsquota:-1, permission:hue supergroup 700}
node:{name:/tmp/hadoop-hue/mapred/staging/hue, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred/staging/hue/.staging, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 700}
node:{name:/tmp/hadoop-hue/mapred/system/jobtracker.info, copys:2, blksize:67108864, blk:[{-7415943897866965603,4,1001}], modtime:1377679253906, acctime:1377679253906, permission:hue supergroup 600}

解析代码如下：

try (DataInputStream in = new DataInputStream(new BufferedInputStream(
				new FileInputStream(fileName)))) {
	System.out.printf("LAYOUT_VERSION:%s\n", in.readInt());
	System.out.printf("namespaceID:%s\n", in.readInt());
	long nodes = in.readLong();
	System.out.printf("numItemsInTree:%s\n", nodes);
	System.out.printf("generationStamp:%s\n\n", in.readLong());

	byte[] byteStore = new byte[4 * FSConstants.MAX_PATH_LENGTH];
	for (long i = 0; i < nodes; i++) {
		printINode(in, byteStore);
		Arrays.fill(byteStore, (byte) 0x00);
	}
}

	void printINode(DataInputStream in, byte[] byteStore)
			throws IOException {
		boolean isDir = true;
		short namelen = in.readShort();
		in.read(byteStore, 0, namelen);
		short replication = in.readShort();
		long modTime = in.readLong();
		long accessTime = in.readLong(); // access time
		long blockSize = in.readLong(); // preferred block size
		int blocks = in.readInt(); // # of blocks
		StringBuilder sb = new StringBuilder();
		long nsQuota = 0;
		long dsQuota = 0;
		if (blocks >= 0) {
			isDir = false;
			sb.append("[");
			for (int i = 0; i < blocks; i++) {
				sb.append("{").append(in.readLong()).append(",")
						.append(in.readLong()).append(",")
						.append(in.readLong()).append("}");
				if (i + 1 != blocks) {
					sb.append(",");
				}
			}
			sb.append("]");
		} else {
			nsQuota = in.readLong();
			dsQuota = in.readLong();
		}
		String userName = Text.readString(in);
		String groupName = Text.readString(in);
		short permit = in.readShort();
		if (isDir) {
			System.out
					.printf("node:{name:%s, modtime:%s, nsquota:%s, dsquota:%s, permission:%s %s %s}\n",
							new String(byteStore, 0, namelen), modTime,
							nsQuota, dsQuota, userName, groupName,
							Integer.toOctalString(permit));
		} else {
			System.out
					.printf("node:{name:%s, copys:%s, blksize:%s, blk:%s, modtime:%s, acctime:%s, permission:%s %s %s}\n",
							new String(byteStore, 0, namelen), replication,
							blockSize, sb.toString(), modTime, accessTime,
							userName, groupName, Integer.toOctalString(permit));
		}

	}