hadoop查看文件大小(hdfs hive hbase)

 

 

 

可以查看 hdfs  hive  hbase文件存储大小

hadoop fs -count 

或者

hdfs dfs -count /51JOB

 

[root@chinadaas01 ~]# hadoop fs -count  /hbase1/zzjg_detail

文件夹数 文件数  这个目录下的文件bytes

 

 

 

查看文件夹下 各个目录的总大小：

 

hdfs dfs -du -h /user/hive/warehouse/ste_qyjs.db/

能看到这个库下所有表的总大小 并以G 或者K 显示

 

 

直接看第三个命令：

 

[bigdata@task1-sandbox tfb]$ hadoop fs  -du -s -h /user/hive/warehouse/dev_treasury.db
1.1 T  3.3 T  /user/hive/warehouse/dev_treasury.db

 

 

 

查看分区表大小：“

 

hadoop fs -du /user/hive/warehouse/dev_treasury.db/ods_common_users|awk '{ SUM += $1 } END { print SUM/(1024*1024*1024)}'

awk处理完所有输入行之后才处理END模式， 上面命令是讲所有分区表和非分区表的大小统计进去 以GB显示

<!--5f39ae17-8c62-4a45-bc43-b32064c9388a:W3siYmxvY2tJZCI6IjEyMjEtMTUyOTM5MzI3MDg1NSIsImJsb2NrVHlwZSI6InBhcmFncmFwaCIsInN0eWxlcyI6eyJhbGlnbiI6ImxlZnQiLCJpbmRlbnQiOjAsInRleHQtaW5kZW50IjowLCJsaW5lLWhlaWdodCI6MS43NX0sInR5cGUiOiJwYXJhZ3JhcGgiLCJyaWNoVGV4dCI6eyJkYXRhIjpbeyJjaGFyIjoiICJ9LHsiY2hhciI6ImgifSx7ImNoYXIiOiJhIn0seyJjaGFyIjoiZCJ9LHsiY2hhciI6Im8ifSx7ImNoYXIiOiJvIn0seyJjaGFyIjoicCJ9LHsiY2hhciI6IiAifSx7ImNoYXIiOiJmIn0seyJjaGFyIjoicyJ9LHsiY2hhciI6IiAifSx7ImNoYXIiOiItIn0seyJjaGFyIjoiZCJ9LHsiY2hhciI6InUifSx7ImNoYXIiOiIgIn0seyJjaGFyIjoiLyJ9LHsiY2hhciI6InUifSx7ImNoYXIiOiJzIn0seyJjaGFyIjoiZSJ9LHsiY2hhciI6InIifSx7ImNoYXIiOiIvIn0seyJjaGFyIjoiaCJ9LHsiY2hhciI6ImkifSx7ImNoYXIiOiJ2In0seyJjaGFyIjoiZSJ9LHsiY2hhciI6Ii8ifSx7ImNoYXIiOiJ3In0seyJjaGFyIjoiYSJ9LHsiY2hhciI6InIifSx7ImNoYXIiOiJlIn0seyJjaGFyIjoiaCJ9LHsiY2hhciI6Im8ifSx7ImNoYXIiOiJ1In0seyJjaGFyIjoicyJ9LHsiY2hhciI6ImUifSx7ImNoYXIiOiIvIn0seyJjaGFyIjoiZCJ9LHsiY2hhciI6ImUifSx7ImNoYXIiOiJ2In0seyJjaGFyIjoiXyJ9LHsiY2hhciI6InQifSx7ImNoYXIiOiJyIn0seyJjaGFyIjoiZSJ9LHsiY2hhciI6ImEifSx7ImNoYXIiOiJzIn0seyJjaGFyIjoidSJ9LHsiY2hhciI6InIifSx7ImNoYXIiOiJ5In0seyJjaGFyIjoiLiJ9LHsiY2hhciI6ImQifSx7ImNoYXIiOiJiIn0seyJjaGFyIjoiLyJ9LHsiY2hhciI6Im8ifSx7ImNoYXIiOiJkIn0seyJjaGFyIjoicyJ9LHsiY2hhciI6Il8ifSx7ImNoYXIiOiJjIn0seyJjaGFyIjoibyJ9LHsiY2hhciI6Im0ifSx7ImNoYXIiOiJtIn0seyJjaGFyIjoibyJ9LHsiY2hhciI6Im4ifSx7ImNoYXIiOiJfIn0seyJjaGFyIjoidSJ9LHsiY2hhciI6InMifSx7ImNoYXIiOiJlIn0seyJjaGFyIjoiciJ9LHsiY2hhciI6InMifSx7ImNoYXIiOiJ8In0seyJjaGFyIjoiYSJ9LHsiY2hhciI6IncifSx7ImNoYXIiOiJrIn0seyJjaGFyIjoiICJ9LHsiY2hhciI6IicifSx7ImNoYXIiOiJ7In0seyJjaGFyIjoiICJ9LHsiY2hhciI6IlMifSx7ImNoYXIiOiJVIn0seyJjaGFyIjoiTSJ9LHsiY2hhciI6IiAifSx7ImNoYXIiOiIrIn0seyJjaGFyIjoiPSJ9LHsiY2hhciI6IiAifSx7ImNoYXIiOiIkIn0seyJjaGFyIjoiMSJ9LHsiY2hhciI6IiAifSx7ImNoYXIiOiJ9In0seyJjaGFyIjoiICJ9LHsiY2hhciI6IkUifSx7ImNoYXIiOiJOIn0seyJjaGFyIjoiRCJ9LHsiY2hhciI6IiAifSx7ImNoYXIiOiJ7In0seyJjaGFyIjoiICJ9LHsiY2hhciI6InAifSx7ImNoYXIiOiJyIn0seyJjaGFyIjoiaSJ9LHsiY2hhciI6Im4ifSx7ImNoYXIiOiJ0In0seyJjaGFyIjoiICJ9LHsiY2hhciI6IlMifSx7ImNoYXIiOiJVIn0seyJjaGFyIjoiTSJ9LHsiY2hhciI6Ii8ifSx7ImNoYXIiOiIoIn0seyJjaGFyIjoiMSJ9LHsiY2hhciI6IjAifSx7ImNoYXIiOiIyIn0seyJjaGFyIjoiNCJ9LHsiY2hhciI6IioifSx7ImNoYXIiOiIxIn0seyJjaGFyIjoiMCJ9LHsiY2hhciI6IjIifSx7ImNoYXIiOiI0In0seyJjaGFyIjoiKiJ9LHsiY2hhciI6IjEifSx7ImNoYXIiOiIwIn0seyJjaGFyIjoiMiJ9LHsiY2hhciI6IjQifSx7ImNoYXIiOiIpIn0seyJjaGFyIjoifSJ9LHsiY2hhciI6IicifV0sImlzUmljaFRleHQiOnRydWUsImtlZXBMaW5lQnJlYWsiOnRydWV9fSx7ImJsb2NrSWQiOiIzNjk0LTE1MjkzOTMyNzE1NDgiLCJibG9ja1R5cGUiOiJwYXJhZ3JhcGgiLCJzdHlsZXMiOnsiYWxpZ24iOiJsZWZ0IiwiaW5kZW50IjowLCJ0ZXh0LWluZGVudCI6MCwibGluZS1oZWlnaHQiOjEuNzV9LCJ0eXBlIjoicGFyYWdyYXBoIiwicmljaFRleHQiOnsiZGF0YSI6W3siY2hhciI6ImEiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoidyIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiJrIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IuWkhCIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLnkIYiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi5a6MIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IuaJgCIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLmnIkiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi6L6TIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IuWFpSIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLooYwiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi5LmLIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IuWQjiIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLmiY0iLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi5aSEIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IueQhiIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiJFIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6Ik4iLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoiRCIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLmqKEiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi5byPIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6Iu+8jCIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiIgIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IuS4iiIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLpnaIiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi5ZG9Iiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IuS7pCIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLmmK8iLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi6K6yIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IuaJgCIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLmnIkiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi5YiGIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IuWMuiIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLooagiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi5ZKMIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IumdniIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLliIYiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi5Yy6Iiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IuihqCIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLnmoQiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi5aSnIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IuWwjyIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLnu58iLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoi6K6hIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6Iui/myIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLljrsiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoiICIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLku6UiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoiRyIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiJCIiwic3R5bGVzIjp7ImZvbnQtc2l6ZSI6MTYsImNvbG9yIjoiIzY2NjY2NiIsImJhY2stY29sb3IiOiIjZmZmZmZmIn19LHsiY2hhciI6IuaYviIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fSx7ImNoYXIiOiLnpLoiLCJzdHlsZXMiOnsiZm9udC1zaXplIjoxNiwiY29sb3IiOiIjNjY2NjY2IiwiYmFjay1jb2xvciI6IiNmZmZmZmYifX0seyJjaGFyIjoiICIsInN0eWxlcyI6eyJmb250LXNpemUiOjE2LCJjb2xvciI6IiM2NjY2NjYiLCJiYWNrLWNvbG9yIjoiI2ZmZmZmZiJ9fV0sImlzUmljaFRleHQiOnRydWUsImtlZXBMaW5lQnJlYWsiOnRydWV9fV0=-->

”

table.txt就是需要校验的hive表名

 

#!/bin/sh
source ~/.bashrc


while read line
do
echo "check table $line begin..."

table=$line
echo "当前统计大小的表是（单位GB）: "$table >> result.txt
hadoop fs -du /user/hive/warehouse/treasury.db/$table|awk '{ SUM += $1 } END { print SUM/(1024*1024*1024)}' >> result.txt